typeclaw 0.46.1 → 0.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +2 -1
  2. package/package.json +2 -2
  3. package/src/agent/tool-file-safety.ts +18 -9
  4. package/src/agent/tools/webfetch/strategies/jq.ts +1 -2
  5. package/src/agent/tools/webfetch/strategies/readability.ts +39 -16
  6. package/src/agent/tools/webfetch/strategies/selector.ts +4 -2
  7. package/src/agent/tools/webfetch/strategies/snapshot.ts +3 -2
  8. package/src/agent/tools/webfetch/tool.ts +3 -3
  9. package/src/bundled-plugins/backup/index.ts +21 -4
  10. package/src/bundled-plugins/backup/runner.ts +19 -5
  11. package/src/bundled-plugins/memory/dreaming.ts +2 -1
  12. package/src/bundled-plugins/memory/load-shards.ts +31 -3
  13. package/src/bundled-plugins/memory/memory-logger.ts +2 -1
  14. package/src/bundled-plugins/memory/memory-snapshot.ts +8 -0
  15. package/src/bundled-plugins/memory/vector/embedder.ts +28 -9
  16. package/src/bundled-plugins/memory/vector/hybrid.ts +10 -2
  17. package/src/bundled-plugins/memory/vector/store.ts +21 -9
  18. package/src/bundled-plugins/security/policies/git-exfil.ts +45 -27
  19. package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +8 -3
  20. package/src/bundled-plugins/security/policies/private-surface-read.ts +6 -0
  21. package/src/channels/adapters/telegram-bot-classify.ts +29 -0
  22. package/src/channels/router.ts +98 -0
  23. package/src/channels/types.ts +7 -0
  24. package/src/cli/command-meta.ts +2 -0
  25. package/src/cli/compose-stats.ts +103 -0
  26. package/src/cli/compose.ts +13 -0
  27. package/src/cli/index.ts +1 -0
  28. package/src/cli/inspect.ts +16 -0
  29. package/src/cli/stats.ts +134 -0
  30. package/src/compose/index.ts +1 -0
  31. package/src/compose/stats.ts +51 -0
  32. package/src/config/config.ts +14 -0
  33. package/src/config/index.ts +4 -0
  34. package/src/container/controller.ts +6 -0
  35. package/src/container/index.ts +1 -0
  36. package/src/container/log-archive.ts +448 -0
  37. package/src/container/shared.ts +49 -19
  38. package/src/container/start.ts +78 -63
  39. package/src/container/stats.ts +86 -0
  40. package/src/container/stop.ts +63 -36
  41. package/src/init/checkpoint.ts +4 -4
  42. package/src/init/dockerfile.ts +87 -88
  43. package/src/init/gitignore.ts +5 -6
  44. package/src/init/restart-deps-preflight.ts +2 -2
  45. package/src/init/run-bun-install.ts +16 -2
  46. package/src/init/validate-api-key.ts +1 -0
  47. package/src/inspect/item-list.ts +12 -6
  48. package/src/permissions/permissions.ts +93 -2
  49. package/src/role-claim/code.ts +9 -5
  50. package/src/run/index.ts +50 -36
  51. package/src/run/resource-report.ts +226 -0
  52. package/src/sandbox/canonical-secrets.ts +7 -0
  53. package/src/sandbox/index.ts +1 -0
  54. package/src/sandbox/symlinks.ts +1 -1
  55. package/src/secrets/export-claude-credentials-file.ts +13 -22
  56. package/src/secrets/export-codex-auth-file.ts +12 -28
  57. package/typeclaw.schema.json +14 -0
package/README.md CHANGED
@@ -79,10 +79,11 @@
79
79
  - 📊 **usage** — reports token and dollar spend by day, model, session, or origin
80
80
  - 🔍 **inspect** — replays a session transcript and tails live activity
81
81
  - 📜 **logs** — streams container logs with local-time prefixes
82
+ - 📈 **stats** — a live cpu / memory / pid snapshot of the agent container
82
83
 
83
84
  ## Compose — manage a fleet from the CLI
84
85
 
85
- - 🎼 **Fleet operations** — discover agent folders and start, stop, restart, check status, tail logs, report usage, and run diagnostics across them from the command line
86
+ - 🎼 **Fleet operations** — discover agent folders and start, stop, restart, check status, snapshot resource stats, tail logs, report usage, and run diagnostics across them from the command line
86
87
 
87
88
  Memory loop and subagent architecture are covered in detail in the [Internals docs](https://typeclaw.dev/docs/internals) and [`src/bundled-plugins/memory/README.md`](./src/bundled-plugins/memory/README.md).
88
89
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "typeclaw",
3
- "version": "0.46.1",
3
+ "version": "0.47.0",
4
4
  "homepage": "https://github.com/typeclaw/typeclaw#readme",
5
5
  "bugs": {
6
6
  "url": "https://github.com/typeclaw/typeclaw/issues"
@@ -50,7 +50,7 @@
50
50
  "@mariozechner/pi-tui": "^0.73.1",
51
51
  "@modelcontextprotocol/sdk": "^1.29.0",
52
52
  "@mozilla/readability": "^0.6.0",
53
- "agent-messenger": "^2.32.1",
53
+ "agent-messenger": "^2.32.3",
54
54
  "cheerio": "^1.2.0",
55
55
  "citty": "^0.2.2",
56
56
  "cron-parser": "^5.5.0",
@@ -861,15 +861,24 @@ async function streamSnapshot(
861
861
  callback(null, chunk)
862
862
  },
863
863
  })
864
- await pipeline(
865
- source.createReadStream({ autoClose: false, start: 0 }),
866
- limiter,
867
- createWriteStream(destination, {
868
- flags: 'wx',
869
- mode: 0o400,
870
- }),
871
- { signal },
872
- )
864
+ // `autoClose: false` leaves the FileHandle to the caller, but the ReadStream
865
+ // opens its OWN fd that `source.close()` never releases and GC never reclaims
866
+ // — one leaked fd per snapshotted file, reaching EMFILE on a long-running
867
+ // agent. Destroy the stream explicitly to free that fd deterministically.
868
+ const readStream = source.createReadStream({ autoClose: false, start: 0 })
869
+ try {
870
+ await pipeline(
871
+ readStream,
872
+ limiter,
873
+ createWriteStream(destination, {
874
+ flags: 'wx',
875
+ mode: 0o400,
876
+ }),
877
+ { signal },
878
+ )
879
+ } finally {
880
+ readStream.destroy()
881
+ }
873
882
  return copied
874
883
  }
875
884
 
@@ -1,5 +1,3 @@
1
- import { raw } from 'jq-wasm'
2
-
3
1
  export class JqError extends Error {
4
2
  constructor(message: string) {
5
3
  super(message)
@@ -17,6 +15,7 @@ export async function applyJq(content: string, query: string): Promise<string> {
17
15
  }
18
16
 
19
17
  try {
18
+ const { raw } = await import('jq-wasm')
20
19
  const result = await raw(parsed as string | object, query)
21
20
  if (result.exitCode !== 0 || result.stderr) {
22
21
  const detail = result.stderr.trim() || `exit code ${result.exitCode}`
@@ -1,19 +1,42 @@
1
- import { Readability } from '@mozilla/readability'
2
- import { JSDOM } from 'jsdom'
3
- import TurndownService from 'turndown'
4
-
5
- const turndown = new TurndownService({
6
- headingStyle: 'atx',
7
- codeBlockStyle: 'fenced',
8
- bulletListMarker: '-',
9
- emDelimiter: '*',
10
- hr: '---',
11
- })
12
- turndown.remove(['script', 'style', 'meta', 'link', 'noscript', 'iframe'])
13
-
14
- type ReadabilityDocument = ConstructorParameters<typeof Readability>[0]
15
-
16
- export function applyReadability(html: string, url: string): string {
1
+ import type { Readability as ReadabilityClass } from '@mozilla/readability'
2
+ import type TurndownService from 'turndown'
3
+
4
+ // Perf: jsdom (+readability+turndown) costs ~40-60MB RSS at import time. Keep it
5
+ // lazy so idle agents that never call web_fetch don't pay for it. Do NOT hoist to
6
+ // a top-level import.
7
+ type ReadabilityDeps = {
8
+ Readability: typeof ReadabilityClass
9
+ JSDOM: typeof import('jsdom').JSDOM
10
+ turndown: TurndownService
11
+ }
12
+
13
+ let depsPromise: Promise<ReadabilityDeps> | undefined
14
+
15
+ async function loadDeps(): Promise<ReadabilityDeps> {
16
+ depsPromise ??= (async () => {
17
+ const [{ Readability }, { JSDOM }, { default: TurndownCtor }] = await Promise.all([
18
+ import('@mozilla/readability'),
19
+ import('jsdom'),
20
+ import('turndown'),
21
+ ])
22
+ const turndown = new TurndownCtor({
23
+ headingStyle: 'atx',
24
+ codeBlockStyle: 'fenced',
25
+ bulletListMarker: '-',
26
+ emDelimiter: '*',
27
+ hr: '---',
28
+ })
29
+ turndown.remove(['script', 'style', 'meta', 'link', 'noscript', 'iframe'])
30
+ return { Readability, JSDOM, turndown }
31
+ })()
32
+ return depsPromise
33
+ }
34
+
35
+ type ReadabilityDocument = ConstructorParameters<typeof ReadabilityClass>[0]
36
+
37
+ export async function applyReadability(html: string, url: string): Promise<string> {
38
+ const { Readability, JSDOM, turndown } = await loadDeps()
39
+
17
40
  const dom = new JSDOM(html, { url })
18
41
  const document = dom.window.document.cloneNode(true) as unknown as ReadabilityDocument
19
42
  const article = new Readability(document).parse()
@@ -1,4 +1,4 @@
1
- import * as cheerio from 'cheerio'
1
+ import type * as cheerio from 'cheerio'
2
2
 
3
3
  export class SelectorError extends Error {
4
4
  constructor(message: string) {
@@ -7,7 +7,9 @@ export class SelectorError extends Error {
7
7
  }
8
8
  }
9
9
 
10
- export function applySelector(html: string, selector: string): string {
10
+ export async function applySelector(html: string, selector: string): Promise<string> {
11
+ const cheerio = await import('cheerio')
12
+
11
13
  let $: cheerio.CheerioAPI
12
14
  try {
13
15
  $ = cheerio.load(html)
@@ -1,4 +1,4 @@
1
- import * as cheerio from 'cheerio'
1
+ import type * as cheerio from 'cheerio'
2
2
  import type { AnyNode, Element } from 'domhandler'
3
3
 
4
4
  const SEMANTIC_TAGS = new Set([
@@ -65,7 +65,8 @@ const ROLE_FOR_TAG: Record<string, string> = {
65
65
  label: 'label',
66
66
  }
67
67
 
68
- export function applySnapshot(html: string): string {
68
+ export async function applySnapshot(html: string): Promise<string> {
69
+ const cheerio = await import('cheerio')
69
70
  const $ = cheerio.load(html)
70
71
  const lines: string[] = []
71
72
  const body = $('body').get(0)
@@ -226,7 +226,7 @@ async function runStrategy(
226
226
  case 'raw':
227
227
  return applyRaw(body)
228
228
  case 'readability':
229
- return applyReadability(body, url)
229
+ return await applyReadability(body, url)
230
230
  case 'jq':
231
231
  try {
232
232
  return await applyJq(body, params.query ?? '')
@@ -236,7 +236,7 @@ async function runStrategy(
236
236
  }
237
237
  case 'selector':
238
238
  try {
239
- return applySelector(body, params.selector ?? '')
239
+ return await applySelector(body, params.selector ?? '')
240
240
  } catch (error) {
241
241
  if (error instanceof SelectorError) throw new Error(error.message)
242
242
  throw error
@@ -255,7 +255,7 @@ async function runStrategy(
255
255
  throw error
256
256
  }
257
257
  case 'snapshot':
258
- return applySnapshot(body)
258
+ return await applySnapshot(body)
259
259
  }
260
260
  }
261
261
 
@@ -299,8 +299,11 @@ async function resolveBackupAuthEnv(
299
299
  }
300
300
  }
301
301
 
302
- export async function resolveOriginPushUrl(cwd: string): Promise<string | null> {
303
- const bun = (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
302
+ export async function resolveOriginPushUrl(
303
+ cwd: string,
304
+ bunOverride?: { spawn: typeof Bun.spawn },
305
+ ): Promise<string | null> {
306
+ const bun = bunOverride ?? (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
304
307
  if (!bun) return null
305
308
  const repo = resolveAgentGit(cwd)
306
309
  if (!repo) return null
@@ -311,8 +314,22 @@ export async function resolveOriginPushUrl(cwd: string): Promise<string | null>
311
314
  stderr: 'ignore',
312
315
  env: { ...process.env, GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0' },
313
316
  })
314
- if ((await proc.exited) !== 0) return null
315
- const out = (await new Response(proc.stdout).text()).trim()
317
+ const exited = proc.exited
318
+ const stdoutDrain = new Response(proc.stdout).text()
319
+ let exitCode: number
320
+ let stdout: string
321
+ try {
322
+ ;[exitCode, stdout] = await Promise.all([exited, stdoutDrain])
323
+ } catch {
324
+ // A stdout-drain rejection must not leave the git child running: kill it
325
+ // and await exit + drain before returning, so `Promise.all`'s early reject
326
+ // can't orphan the subprocess or its pipe.
327
+ proc.kill()
328
+ await Promise.allSettled([exited, stdoutDrain])
329
+ return null
330
+ }
331
+ if (exitCode !== 0) return null
332
+ const out = stdout.trim()
316
333
  return out === '' ? null : out
317
334
  } catch {
318
335
  return null
@@ -399,9 +399,9 @@ function sanitizeCommitMessage(raw: string): string {
399
399
  return rest.length > 0 ? `${subject}\n\n${rest}` : subject
400
400
  }
401
401
 
402
- export function makeDefaultGitSpawn(): GitSpawn {
402
+ export function makeDefaultGitSpawn(bunOverride?: { spawn: typeof Bun.spawn }): GitSpawn {
403
403
  return withIndexLockRetry(async (args, { cwd, timeoutMs, env }) => {
404
- const bun = (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
404
+ const bun = bunOverride ?? (globalThis as { Bun?: { spawn: typeof Bun.spawn } }).Bun
405
405
  if (!bun) {
406
406
  return { exitCode: 127, stdout: '', stderr: 'Bun runtime not available', timedOut: false }
407
407
  }
@@ -419,9 +419,23 @@ export function makeDefaultGitSpawn(): GitSpawn {
419
419
  env: { ...process.env, ...NONINTERACTIVE_ENV, ...env },
420
420
  signal: controller.signal,
421
421
  })
422
- const exitCode = await proc.exited
423
- const stdout = await new Response(proc.stdout).text()
424
- const stderr = await new Response(proc.stderr).text()
422
+ const exitedDrain = proc.exited
423
+ const stdoutDrain = new Response(proc.stdout).text()
424
+ const stderrDrain = new Response(proc.stderr).text()
425
+ let exitCode: number
426
+ let stdout: string
427
+ let stderr: string
428
+ try {
429
+ ;[exitCode, stdout, stderr] = await Promise.all([exitedDrain, stdoutDrain, stderrDrain])
430
+ } catch (drainErr) {
431
+ // A pipe-read rejection must not escape while the git child is still
432
+ // running (the outer `finally` would clear the abort timer and leave a
433
+ // stuck child unbounded). Abort to kill it, then wait for exit and both
434
+ // drains to settle before surfacing the error.
435
+ controller.abort()
436
+ await Promise.allSettled([exitedDrain, stdoutDrain, stderrDrain])
437
+ throw drainErr
438
+ }
425
439
  const timedOut = controller.signal.aborted
426
440
  return { exitCode, stdout, stderr, timedOut }
427
441
  } catch (err) {
@@ -26,6 +26,7 @@ import {
26
26
  } from './dreaming-state'
27
27
  import { parseShard, renderShard, type ShardFrontmatter } from './frontmatter'
28
28
  import { listShardSlugs, loadAllShards, loadShard, type TopicShard } from './load-shards'
29
+ import { formatMemorySnapshot } from './memory-snapshot'
29
30
  import { referencesDir, streamFilePath, streamsDir, topicShardPath, topicsDir } from './paths'
30
31
  import { renderReference } from './references/frontmatter'
31
32
  import { loadAllReferences, type Reference } from './references/load-references'
@@ -1477,7 +1478,7 @@ export function createDreamingSubagent(options: CreateDreamingSubagentOptions =
1477
1478
  try {
1478
1479
  await commit(ctx.payload.agentDir)
1479
1480
  logger.info(
1480
- `[dreaming] done topics_created=${metrics.topicsCreated} topics_removed=${metrics.topicsRemoved} superseded_new=${metrics.supersededDelta} fragments_dropped=${compaction.fragmentsDropped} over_budget=${overBudget.length} references_demoted=${referencesDemoted} references_evicted=${referencesEvicted} elapsed_ms=${Date.now() - start}`,
1481
+ `[dreaming] done topics_created=${metrics.topicsCreated} topics_removed=${metrics.topicsRemoved} superseded_new=${metrics.supersededDelta} fragments_dropped=${compaction.fragmentsDropped} over_budget=${overBudget.length} references_demoted=${referencesDemoted} references_evicted=${referencesEvicted} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
1481
1482
  )
1482
1483
  } catch (err) {
1483
1484
  const message = err instanceof Error ? err.message : String(err)
@@ -1,4 +1,4 @@
1
- import { readdir, readFile, stat } from 'node:fs/promises'
1
+ import { mkdir, readdir, readFile, stat } from 'node:fs/promises'
2
2
 
3
3
  import { parseShard, type ShardFrontmatter } from './frontmatter'
4
4
  import { topicShardPath, topicsDir } from './paths'
@@ -117,11 +117,23 @@ export async function loadShard(
117
117
  }
118
118
 
119
119
  export async function listShardSlugs(agentDir: string): Promise<string[]> {
120
+ const dir = topicsDir(agentDir)
120
121
  let names: string[]
121
122
  try {
122
- names = await readdir(topicsDir(agentDir))
123
+ names = await readdir(dir)
123
124
  } catch (err) {
124
- if (isEnoent(err)) return []
125
+ // ENOENT: the topics dir doesn't exist yet (fresh agent, or a wiped memory
126
+ // tree). Self-heal by recreating it so every downstream reader -- per-turn
127
+ // injection, memory_search, dreaming, the startup index build, and the
128
+ // doctor vector-index check -- sees a real, empty directory instead of a
129
+ // missing one. mkdir is idempotent, so a lost race just no-ops.
130
+ if (isEnoent(err)) return healMissingTopicsDir(dir)
131
+ // ENOTDIR: a bind-mount/tmpfs coherency hiccup can momentarily replace the
132
+ // topics dir (or a parent path component) with a non-directory; readdir
133
+ // then throws ENOTDIR. We can't self-heal this -- mkdir over a regular file
134
+ // would either fail or clobber operator data -- so degrade to empty and let
135
+ // the transient host state resolve itself, exactly as before.
136
+ if (isEnotdir(err)) return []
125
137
  throw err
126
138
  }
127
139
 
@@ -131,6 +143,18 @@ export async function listShardSlugs(agentDir: string): Promise<string[]> {
131
143
  .sort()
132
144
  }
133
145
 
146
+ async function healMissingTopicsDir(dir: string): Promise<string[]> {
147
+ try {
148
+ await mkdir(dir, { recursive: true })
149
+ } catch (err) {
150
+ // A concurrent healer already created it, or the parent briefly became a
151
+ // non-directory. Either way the empty-list contract still holds; never let
152
+ // the heal attempt turn a self-healing state into a hard error.
153
+ if (!isEnotdir(err)) throw err
154
+ }
155
+ return []
156
+ }
157
+
134
158
  // Test-only helper. Clears the in-memory shard cache so tests that exercise
135
159
  // the cache invalidation path can simulate a cold start without spinning up a
136
160
  // fresh process.
@@ -180,3 +204,7 @@ function getOrCreateCache(agentDir: string): Map<string, ShardCacheEntry> {
180
204
  function isEnoent(err: unknown): boolean {
181
205
  return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOENT'
182
206
  }
207
+
208
+ function isEnotdir(err: unknown): boolean {
209
+ return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOTDIR'
210
+ }
@@ -6,6 +6,7 @@ import { formatLocalDate } from '@/shared'
6
6
 
7
7
  import { advanceWatermarkTool, createAppendTool, type FragmentsAppendedHook } from './append-tool'
8
8
  import { findEntryTool } from './find-entry-tool'
9
+ import { formatMemorySnapshot } from './memory-snapshot'
9
10
  import { streamFilePath, streamsDir } from './paths'
10
11
  import { listReferenceSlugs } from './references/load-references'
11
12
  import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
@@ -352,7 +353,7 @@ export function createMemoryLoggerSubagent(
352
353
  await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark) })
353
354
  const fragmentsWritten = (await countFragments(streamFile)) - fragmentsBefore
354
355
  logger.info(
355
- `[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start}`,
356
+ `[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
356
357
  )
357
358
  } catch (err) {
358
359
  const message = err instanceof Error ? err.message : String(err)
@@ -0,0 +1,8 @@
1
+ export type MemorySample = Pick<NodeJS.MemoryUsage, 'rss' | 'heapUsed' | 'heapTotal' | 'external' | 'arrayBuffers'>
2
+
3
+ const BYTES_PER_MB = 1024 * 1024
4
+
5
+ export function formatMemorySnapshot(sample: MemorySample = process.memoryUsage()): string {
6
+ const mb = (bytes: number): number => Math.round(bytes / BYTES_PER_MB)
7
+ return `rss_mb=${mb(sample.rss)} heapUsed_mb=${mb(sample.heapUsed)} external_mb=${mb(sample.external)} arrayBuffers_mb=${mb(sample.arrayBuffers)}`
8
+ }
@@ -6,6 +6,7 @@ import { join } from 'node:path'
6
6
  // values are pulled lazily via `loadTransformers()` below.
7
7
  import type { env as TransformersEnvValue, pipeline as TransformersPipeline } from '@huggingface/transformers'
8
8
 
9
+ import { createKeyedSemaphore } from '@/agent/tools/keyed-semaphore'
9
10
  import {
10
11
  assertModelCacheCompatible,
11
12
  EMBEDDING_DIMS,
@@ -16,6 +17,7 @@ import {
16
17
  import { getResolvedTransformersVersion } from '@/models/transformers-version'
17
18
 
18
19
  import { homeRoot } from '../../../hostd/paths'
20
+ import { formatMemorySnapshot } from '../memory-snapshot'
19
21
  import { resolveEmbedThreadCount } from './embed-threads'
20
22
  import { type BoundedText, boundEmbeddableText, MAX_MODEL_TOKENS } from './truncation'
21
23
 
@@ -36,6 +38,21 @@ export type EmbedType = 'query' | 'passage'
36
38
  // loaded once and reused across chunks, so the only cost is sequential passes.
37
39
  const EMBED_BATCH_SIZE = 64
38
40
 
41
+ // Concurrent embeds inside inference at once, process-wide. The intra-op thread
42
+ // cap (embed-threads.ts) bounds ONE embed's footprint, but nothing bounded how
43
+ // MANY embeds run at once: a busy channel turn's hybridSearch, a dreaming pass,
44
+ // and a memory-logger append can all be inside the shared ONNX session
45
+ // simultaneously (concurrent `Run()` is allowed on the Node backend), stacking
46
+ // their activation memory into one RSS spike that the OOM killer takes with a
47
+ // bare SIGKILL. This caps that stack. Chosen 2, not 1: the workload is
48
+ // latency-sensitive per-turn query embeds, so a hard serial would park a live
49
+ // turn behind a whole dreaming batch; 2 keeps one lane free while still bounding
50
+ // the worst-case multiplier. Single fixed key — the limit is global, not
51
+ // per-key.
52
+ const EMBED_CONCURRENCY = 2
53
+ const EMBED_SEMAPHORE_KEY = 'embed'
54
+ const embedSemaphore = createKeyedSemaphore({ concurrency: EMBED_CONCURRENCY })
55
+
39
56
  type TransformersEnv = typeof TransformersEnvValue
40
57
  type FeatureExtractor = Awaited<ReturnType<typeof TransformersPipeline<'feature-extraction'>>>
41
58
 
@@ -154,14 +171,16 @@ export class Embedder {
154
171
  // upserts) finish in a pass or two and would only spam the logs.
155
172
  const reportProgress = prefixed.length >= LARGE_EMBED
156
173
 
157
- const embeddings: Float32Array[] = []
158
- for (let start = 0; start < prefixed.length; start += EMBED_BATCH_SIZE) {
159
- const batch = prefixed.slice(start, start + EMBED_BATCH_SIZE)
160
- const output = await this.extractor(batch, { pooling: 'mean', normalize: true })
161
- embeddings.push(...toEmbeddings(output.data, batch.length))
162
- if (reportProgress) logEmbedProgress(embeddings.length, prefixed.length, type)
163
- }
164
- return embeddings
174
+ return embedSemaphore.run(EMBED_SEMAPHORE_KEY, async () => {
175
+ const embeddings: Float32Array[] = []
176
+ for (let start = 0; start < prefixed.length; start += EMBED_BATCH_SIZE) {
177
+ const batch = prefixed.slice(start, start + EMBED_BATCH_SIZE)
178
+ const output = await this.extractor(batch, { pooling: 'mean', normalize: true })
179
+ embeddings.push(...toEmbeddings(output.data, batch.length))
180
+ if (reportProgress) logEmbedProgress(embeddings.length, prefixed.length, type)
181
+ }
182
+ return embeddings
183
+ })
165
184
  }
166
185
  }
167
186
 
@@ -213,7 +232,7 @@ function warnIfBounded(results: readonly BoundedText[], type: EmbedType): void {
213
232
  const LARGE_EMBED = 256
214
233
 
215
234
  function logEmbedBatch(count: number, type: EmbedType): void {
216
- const line = `[memory] vector embedding: ${count} ${type} input(s) (chunked at ${EMBED_BATCH_SIZE}/pass)`
235
+ const line = `[memory] vector embedding: ${count} ${type} input(s) (chunked at ${EMBED_BATCH_SIZE}/pass) ${formatMemorySnapshot()}`
217
236
  if (count >= LARGE_EMBED) {
218
237
  console.info(`${line} — large build, this may take a while`)
219
238
  } else {
@@ -24,7 +24,7 @@ import { embed, EMBEDDING_MODEL_ID, type EmbedType } from './embedder'
24
24
  import type { Passage } from './passages'
25
25
  import { clearsBaseline, gateRelevance, MARGIN, streamAdmissionBaseline } from './relevance-gate'
26
26
  import { crossScriptMarginScale, dominantScript, type ScriptClass } from './script'
27
- import { VectorStore, type ScoredVectorRow, type VectorRow } from './store'
27
+ import { scoreRows, VectorStore, type ScoredVectorRow, type VectorRow } from './store'
28
28
  import { chunkEmbeddableText } from './truncation'
29
29
 
30
30
  export { collectPassages, findMissingPassages, type Passage } from './passages'
@@ -241,9 +241,17 @@ function maxScoreAcrossChunks(
241
241
  queryEmbeddings: Float32Array[],
242
242
  store: VectorStore,
243
243
  ): { scored: ScoredVectorRow[]; winnerChunkByRowId: Map<string, number> } {
244
+ // Decode the compatible corpus ONCE and score every chunk against it, instead
245
+ // of re-reading and re-decoding the whole table per chunk (queryScored did a
246
+ // full SELECT + BLOB decode on each call). Same MAX-across-chunks result, but
247
+ // the O(rows) decode runs once per turn instead of once per chunk.
248
+ const dims = queryEmbeddings[0]?.length
249
+ if (dims === undefined) return { scored: [], winnerChunkByRowId: new Map() }
250
+ const rows = store.loadRows(EMBEDDING_MODEL_ID, dims)
251
+
244
252
  const best = new Map<string, { scored: ScoredVectorRow; chunkIndex: number }>()
245
253
  queryEmbeddings.forEach((embedding, chunkIndex) => {
246
- for (const scoredRow of store.queryScored(embedding, EMBEDDING_MODEL_ID)) {
254
+ for (const scoredRow of scoreRows(rows, embedding)) {
247
255
  const existing = best.get(scoredRow.row.id)
248
256
  if (existing === undefined || scoredRow.score > existing.scored.score) {
249
257
  best.set(scoredRow.row.id, { scored: scoredRow, chunkIndex })
@@ -95,24 +95,28 @@ export class VectorStore {
95
95
  // Same cosine scan as `query` but returns every compatible row WITH its score
96
96
  // and unsliced, so a caller can reason about the full score distribution (the
97
97
  // relevance gate's per-query baseline) before deciding how many to keep.
98
+ queryScored(embedding: Float32Array, modelId: string): ScoredVectorRow[] {
99
+ return scoreRows(this.loadRows(modelId, embedding.length), embedding).sort((a, b) => b.score - a.score)
100
+ }
101
+
102
+ // Decodes every compatible row's embedding BLOB into a Float32Array ONCE. A
103
+ // multi-chunk query scores many embeddings against the SAME corpus, so callers
104
+ // that decode here once and reuse the result across chunks avoid re-reading and
105
+ // re-decoding the whole table per chunk (the per-turn allocation hot path). The
106
+ // returned rows are transient — held only for the current search, never cached
107
+ // across turns (that would trade collectable garbage for permanent resident
108
+ // memory, the opposite of what we want).
98
109
  //
99
110
  // Filter by embedding identity, not dims alone: a stale row from a different
100
111
  // model/dtype variant can share the same dims but lives in an incompatible
101
112
  // vector space, so cosine against it is garbage. Excluding it here keeps a
102
113
  // partial re-embed (mixed variants mid-rebuild) at reduced recall, never
103
114
  // wrong scores.
104
- queryScored(embedding: Float32Array, modelId: string): ScoredVectorRow[] {
105
- // The query vector's magnitude is identical for every row in this scan, so
106
- // hoist it out of the per-row cosine instead of recomputing N times (each
107
- // recompute is 768 multiply-adds + a sqrt). Behavior is unchanged — same
108
- // cosine values, fewer operations on the brute-force hot path.
109
- const queryMagnitude = magnitude(embedding)
115
+ loadRows(modelId: string, dims: number): VectorRow[] {
110
116
  return this.db
111
117
  .query<StoredVectorRow, [string, number]>('SELECT * FROM vectors WHERE model = ? AND dims = ?')
112
- .all(modelId, embedding.length)
118
+ .all(modelId, dims)
113
119
  .map(toVectorRow)
114
- .map((row) => ({ row, score: cosineSimilarity(embedding, queryMagnitude, row.embedding) }))
115
- .sort((a, b) => b.score - a.score)
116
120
  }
117
121
 
118
122
  deleteOtherModels(modelId: string): void {
@@ -160,6 +164,14 @@ export class VectorStore {
160
164
  }
161
165
  }
162
166
 
167
+ // Scores pre-decoded rows against one query embedding. The query vector's
168
+ // magnitude is identical for every row, so hoist it out of the per-row cosine
169
+ // instead of recomputing N times (each recompute is 768 multiply-adds + a sqrt).
170
+ export function scoreRows(rows: VectorRow[], embedding: Float32Array): ScoredVectorRow[] {
171
+ const queryMagnitude = magnitude(embedding)
172
+ return rows.map((row) => ({ row, score: cosineSimilarity(embedding, queryMagnitude, row.embedding) }))
173
+ }
174
+
163
175
  function toVectorRow(row: StoredVectorRow): VectorRow {
164
176
  return {
165
177
  id: row.id,
@@ -3,33 +3,43 @@ import { ACKNOWLEDGE_GUARDS, type SecurityBlock, isGuardAcknowledged } from '../
3
3
  import { getRemoteTaint, recordRemoteTaint } from './remote-taint-state'
4
4
 
5
5
  export const GUARD_GIT_EXFIL = 'gitExfil'
6
+ // The coarse, role-gated first-line publication/file-transfer gate. It matches
7
+ // the command STRING only (see checkGitExfilGuard); it has no working
8
+ // directory, no resolved git repository identity, and no view of a checkout's
9
+ // content provenance. It therefore fires regardless of where the command runs
10
+ // — a `/tmp` clone is NOT a trusted boundary, because an earlier turn may have
11
+ // copied identity files or secrets into it, and the effective repository is
12
+ // unknowable from the string alone (`git -C`, `--git-dir`, `GIT_DIR`, `cd`,
13
+ // subshells, aliases). Do NOT re-introduce cwd/path scoping here: it would be
14
+ // a bypass, not a fix (Oracle-reviewed). Member-tier publishing, if ever
15
+ // required, belongs in a separate out-of-band capability, not a path heuristic.
16
+ //
6
17
  // Classified `medium` (silent-attack axis). Originally `high`; reclassified
7
- // because the actual audience-leak surface for `git push` lives in
8
- // `gitRemoteTainted`, not here. A `git push` to a CLEAN, operator-configured
9
- // remote is not audience-leak — the audience (the remote git host) was
10
- // chosen by the operator and is inside their perimeter. The breach pattern
11
- // PR #134 was written for (re-point origin to attacker URL, then push) is
12
- // gated by `gitRemoteTainted` (still high). The recorder-vs-checker split
13
- // is what makes this reclassification safe: the recorder fires for any
14
- // actor who can run a `git remote set-url` (per-guard bypass OR the
15
- // medium-tier permission via the OR check), so trusted's first-step
16
- // set-url still records taint and the second-step push still gets caught
17
- // by `gitRemoteTainted` even though trusted no longer needs to ack the
18
- // push itself. Net effect: trusted users can push to remotes the operator
19
- // configured without per-call acks, but cannot retarget-and-push.
18
+ // because the one narrow retarget-then-push audience-leak shape (re-point
19
+ // origin to an attacker URL, then push) is gated separately by
20
+ // `gitRemoteTainted` (still high). The recorder-vs-checker split is what makes
21
+ // that reclassification safe: the recorder fires for any actor who can run a
22
+ // `git remote set-url` (per-guard bypass OR the medium-tier permission via the
23
+ // OR check), so trusted's first-step set-url still records taint and the
24
+ // second-step push still gets caught by `gitRemoteTainted` even though trusted
25
+ // no longer needs to ack the push itself. Net effect: trusted users can push
26
+ // to remotes the operator configured without per-call acks, but cannot
27
+ // retarget-and-push.
20
28
  export const GUARD_GIT_EXFIL_SEVERITY: SecuritySeverity = 'medium'
21
29
  export const GUARD_GIT_REMOTE_TAINTED = 'gitRemoteTainted'
22
- // Classified `high` (audience-leak axis): the actual audience-leak gate
23
- // for git. A push after a mid-session `git remote set-url` to an
24
- // attacker-controlled URL is exactly the breach pattern that motivated
25
- // the entire security plugin per PR #134. Stays high regardless of how
26
- // `gitExfil` is classified — the two are independent per-guard strings
27
- // AND independent tier classifications. The recorder-vs-checker split
28
- // (see comment on recordGitRemoteTaintIfAny below) is still load-bearing:
29
- // the recorder fires for anyone who can run the underlying `set-url`
30
- // command (ack, per-guard `bypassGitExfil`, OR the medium-tier permission
31
- // — which now includes trusted by default), so the second-step taint
32
- // check still fires on the eventual push.
30
+ // Classified `high` (audience-leak axis). This is NOT the complete audience-
31
+ // leak defense for git — it gates ONE specific two-step shape: a push to a
32
+ // remote whose URL was changed mid-session (the breach pattern that motivated
33
+ // the security plugin per PR #134). Its state is keyed by session + remote
34
+ // name, so it does not cover pushes to an initially-malicious remote, direct
35
+ // URL pushes, or destinations established outside its session recorder — those
36
+ // stay under the coarse `gitExfil` gate above. Stays high regardless of how
37
+ // `gitExfil` is classified — the two are independent per-guard strings AND
38
+ // independent tier classifications. The recorder-vs-checker split (see comment
39
+ // on recordGitRemoteTaintIfAny below) is still load-bearing: the recorder
40
+ // fires for anyone who can run the underlying `set-url` command (ack, per-guard
41
+ // `bypassGitExfil`, OR the medium-tier permission — which now includes trusted
42
+ // by default), so the second-step taint check still fires on the eventual push.
33
43
  export const GUARD_GIT_REMOTE_TAINTED_SEVERITY: SecuritySeverity = 'high'
34
44
 
35
45
  // Anchors we reuse: a `git` token must be at start-of-line or follow a shell
@@ -185,9 +195,17 @@ export function checkGitExfilGuard(options: {
185
195
  return {
186
196
  block: true,
187
197
  reason: [
188
- `Guard \`${GUARD_GIT_EXFIL}\` blocked bash command that looks like agent-folder exfiltration: ${matched.label}.`,
189
- // kept: pre-migration agents may still have a root MEMORY.md.
190
- 'Pushing a repo, adding a remote, or piping a remote payload to a shell can leak identity files (MEMORY.md, IDENTITY.md, SOUL.md, AGENTS.md) and embedded secrets to attacker-controlled infrastructure - including via prompt-injected requests from chat channels.',
198
+ `Guard \`${GUARD_GIT_EXFIL}\` blocked a risky repository-publication or file-transfer command: ${matched.label}.`,
199
+ // Deliberately directory-INDEPENDENT: the guard sees only the command
200
+ // string and cannot prove which repository git will operate on, nor that
201
+ // an external checkout (e.g. a /tmp clone) does not already contain data
202
+ // copied out of the agent folder. A "clean" path is not a security
203
+ // boundary, so this fires wherever the command runs. Pushing a repo,
204
+ // adding/re-pointing a remote, or piping a remote payload to a shell can
205
+ // publish identity files (MEMORY.md, IDENTITY.md, SOUL.md, AGENTS.md) and
206
+ // embedded secrets - including via prompt-injected requests from chat
207
+ // channels.
208
+ 'This applies regardless of working directory.',
191
209
  `If this is genuinely intentional and the user (not a channel message) explicitly asked for it, retry with \`${ACKNOWLEDGE_GUARDS}.${GUARD_GIT_EXFIL}: true\` in the bash arguments.`,
192
210
  ].join(' '),
193
211
  }