typeclaw 0.46.2 → 0.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +1 -1
- package/src/agent/git-nudge.ts +4 -12
- package/src/agent/session-origin.ts +8 -6
- package/src/agent/tools/channel-reply.ts +13 -13
- package/src/agent/tools/channel-send.ts +3 -3
- package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +4 -11
- package/src/bundled-plugins/memory/dreaming.ts +28 -67
- package/src/bundled-plugins/memory/load-shards.ts +31 -3
- package/src/bundled-plugins/memory/memory-logger.ts +2 -1
- package/src/bundled-plugins/memory/memory-snapshot.ts +8 -0
- package/src/bundled-plugins/memory/vector/embedder.ts +28 -9
- package/src/bundled-plugins/security/policies/git-exfil.ts +45 -27
- package/src/channels/adapters/telegram-bot-classify.ts +29 -0
- package/src/channels/continuation-willingness.ts +2 -2
- package/src/channels/github-false-receipt.ts +4 -4
- package/src/channels/github-rereview-guard.ts +8 -8
- package/src/channels/router.ts +276 -58
- package/src/channels/types.ts +7 -0
- package/src/cli/command-meta.ts +2 -0
- package/src/cli/compose-stats.ts +103 -0
- package/src/cli/compose.ts +13 -0
- package/src/cli/index.ts +1 -0
- package/src/cli/stats.ts +134 -0
- package/src/compose/index.ts +1 -0
- package/src/compose/stats.ts +51 -0
- package/src/config/config.ts +14 -0
- package/src/config/index.ts +4 -0
- package/src/container/controller.ts +6 -0
- package/src/container/index.ts +1 -0
- package/src/container/log-archive.ts +448 -0
- package/src/container/shared.ts +30 -13
- package/src/container/start.ts +78 -63
- package/src/container/stats.ts +86 -0
- package/src/container/stop.ts +63 -36
- package/src/git/run.ts +60 -0
- package/src/init/restart-deps-preflight.ts +2 -2
- package/src/run/index.ts +5 -0
- package/src/run/resource-report.ts +226 -0
- package/src/sandbox/canonical-secrets.ts +1 -0
- package/src/skills/typeclaw-channel-github/SKILL.md +4 -4
- package/typeclaw.schema.json +14 -0
package/README.md
CHANGED
|
@@ -79,10 +79,11 @@
|
|
|
79
79
|
- 📊 **usage** — reports token and dollar spend by day, model, session, or origin
|
|
80
80
|
- 🔍 **inspect** — replays a session transcript and tails live activity
|
|
81
81
|
- 📜 **logs** — streams container logs with local-time prefixes
|
|
82
|
+
- 📈 **stats** — a live cpu / memory / pid snapshot of the agent container
|
|
82
83
|
|
|
83
84
|
## Compose — manage a fleet from the CLI
|
|
84
85
|
|
|
85
|
-
- 🎼 **Fleet operations** — discover agent folders and start, stop, restart, check status, tail logs, report usage, and run diagnostics across them from the command line
|
|
86
|
+
- 🎼 **Fleet operations** — discover agent folders and start, stop, restart, check status, snapshot resource stats, tail logs, report usage, and run diagnostics across them from the command line
|
|
86
87
|
|
|
87
88
|
Memory loop and subagent architecture are covered in detail in the [Internals docs](https://typeclaw.dev/docs/internals) and [`src/bundled-plugins/memory/README.md`](./src/bundled-plugins/memory/README.md).
|
|
88
89
|
|
package/package.json
CHANGED
package/src/agent/git-nudge.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { hooklessGitArgs } from '@/git/hookless'
|
|
2
1
|
import { resolveAgentGit } from '@/git/resolve-agent-git'
|
|
2
|
+
import { runGit } from '@/git/run'
|
|
3
3
|
|
|
4
4
|
const MAX_LISTED_PATHS = 10
|
|
5
5
|
|
|
@@ -65,22 +65,14 @@ function filterAgentOwned(paths: readonly string[]): string[] {
|
|
|
65
65
|
return paths.filter((p) => !RUNTIME_OWNED_PREFIXES.some((prefix) => p.startsWith(prefix)))
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
// Mirrors the spawn pattern in `src/container/start.ts` `commitSystemFile`.
|
|
69
68
|
const defaultDeps: GitNudgeDeps = {
|
|
70
69
|
async readStatus(agentDir, gitArgs) {
|
|
71
70
|
const bun = getBun()
|
|
72
71
|
if (!bun) return null
|
|
73
72
|
try {
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
stdout: 'pipe',
|
|
78
|
-
stderr: 'pipe',
|
|
79
|
-
})
|
|
80
|
-
const exit = await proc.exited
|
|
81
|
-
if (exit !== 0) return null
|
|
82
|
-
const text = await new Response(proc.stdout).text()
|
|
83
|
-
return parsePorcelain(text)
|
|
73
|
+
const { exitCode, stdout } = await runGit(bun, agentDir, [...gitArgs, 'status', '--porcelain=v1'])
|
|
74
|
+
if (exitCode !== 0) return null
|
|
75
|
+
return parsePorcelain(stdout)
|
|
84
76
|
} catch {
|
|
85
77
|
return null
|
|
86
78
|
}
|
|
@@ -486,15 +486,17 @@ function renderChannelOrigin(
|
|
|
486
486
|
...(origin.adapter === 'github'
|
|
487
487
|
? [
|
|
488
488
|
'tool call, keep working and post the answer with a single final',
|
|
489
|
-
'`channel_reply
|
|
489
|
+
'`channel_reply({ text, more_work_this_turn: false })`.',
|
|
490
|
+
'Do not post an "On it" ack comment first — the runtime',
|
|
490
491
|
'already added an :eyes: reaction; use `channel_react` for explicit ack.',
|
|
491
492
|
]
|
|
492
493
|
: [
|
|
493
494
|
'tool call, send a one-line ack first via `channel_reply({ text: "On it.",',
|
|
494
|
-
'
|
|
495
|
-
'`channel_reply
|
|
496
|
-
'
|
|
497
|
-
'the ack
|
|
495
|
+
'more_work_this_turn: true })`, keep working, then send the answer with a',
|
|
496
|
+
'final `channel_reply({ text, more_work_this_turn: false })`.',
|
|
497
|
+
'The ack is not your reply; the answer is. Once the answer lands, end your turn.',
|
|
498
|
+
'`more_work_this_turn: true` is mandatory on the ack or the turn ends there',
|
|
499
|
+
'and drops the fetch/subagent/actual answer.',
|
|
498
500
|
]),
|
|
499
501
|
'',
|
|
500
502
|
'**Backgrounded work does not end the obligation.** If you spawn a',
|
|
@@ -508,7 +510,7 @@ function renderChannelOrigin(
|
|
|
508
510
|
'',
|
|
509
511
|
'Do not send a second reply just to rephrase, restate, or "confirm" what you already said.',
|
|
510
512
|
'',
|
|
511
|
-
'To reply here, call `channel_reply({ text })`. Addressing (including the',
|
|
513
|
+
'To reply here, call `channel_reply({ text, more_work_this_turn })`. Addressing (including the',
|
|
512
514
|
`thread${origin.thread !== null ? '' : ' — none here, this is a channel-root session'}) is filled in; you don't need`,
|
|
513
515
|
'to copy these fields:',
|
|
514
516
|
'',
|
|
@@ -83,17 +83,17 @@ export function createChannelReplyTool({
|
|
|
83
83
|
},
|
|
84
84
|
),
|
|
85
85
|
),
|
|
86
|
-
|
|
86
|
+
more_work_this_turn: Type.Boolean({
|
|
87
87
|
description:
|
|
88
|
-
'REQUIRED on every channel_reply — you must explicitly choose, there is no default. Set `true` when this reply is a mid-turn status update (e.g. "working on it…") and you still have
|
|
88
|
+
'REQUIRED on every channel_reply — you must explicitly choose, there is no default. Set `true` when this reply is a mid-turn status update (e.g. "working on it…") and you still have MORE WORK to do THIS turn — fetching data, running a tool, spawning a subagent, then replying again; `true` keeps the turn alive so that follow-up actually runs. ' +
|
|
89
89
|
'Set `false` when this reply is your final message for the turn (the common case). ' +
|
|
90
|
-
'This choice is mandatory precisely because a missing value used to default to ending the turn silently: a successful reply ends the turn unless `
|
|
90
|
+
'This choice is mandatory precisely because a missing value used to default to ending the turn silently: a successful reply ends the turn unless `more_work_this_turn` is `true`, so a `false` on an ack you meant to keep working from drops the work you promised. ' +
|
|
91
91
|
'Do not set `true` just to seem responsive; only when genuine multi-step work follows in the same turn.',
|
|
92
92
|
}),
|
|
93
93
|
resolve_review_thread: Type.Optional(
|
|
94
94
|
Type.Boolean({
|
|
95
95
|
description:
|
|
96
|
-
'GitHub PR review threads ONLY. On a TERMINAL (`
|
|
96
|
+
'GitHub PR review threads ONLY. On a TERMINAL (`more_work_this_turn:false`) github PR review-thread reply that carries `text`, this is REQUIRED: you must set it to `true` or `false` and omitting it is rejected — you will be told to re-call with an explicit choice. (It stays a normal optional field everywhere else: ignored on Slack/Discord/Telegram/KakaoTalk and any non-github session, on github replies outside a review thread, on attachments-only replies, and on mid-turn `more_work_this_turn:true` status updates — leave it unset there.) ' +
|
|
97
97
|
'Set `true` when your `text` acknowledges the concern is fixed/verified/addressed (e.g. "verified at <sha>", "thanks, that resolves it"): the runtime resolves the thread BEFORE posting, so the close-out actually happens in this same turn — a successful reply ends the turn, so a resolve deferred to "later" never runs. ' +
|
|
98
98
|
"Set `false` when the thread should stay open (partial fix, disagreement, mid-discussion). It is safe to set `true` by default: the runtime resolves ONLY if the thread's root comment is yours — it refuses (and blocks the reply) on a human reviewer's thread, so you never close someone else's open question. You need not pre-check authorship; let the runtime enforce ownership.",
|
|
99
99
|
}),
|
|
@@ -103,7 +103,7 @@ export function createChannelReplyTool({
|
|
|
103
103
|
async execute(_toolCallId, params) {
|
|
104
104
|
let text = params.text
|
|
105
105
|
const attachments = params.attachments
|
|
106
|
-
const keepTurnAlive = params.
|
|
106
|
+
const keepTurnAlive = params.more_work_this_turn === true
|
|
107
107
|
if ((text === undefined || text === '') && (attachments === undefined || attachments.length === 0)) {
|
|
108
108
|
logger.warn(formatChannelToolFailure('channel_reply', 'missing text and attachments'))
|
|
109
109
|
return {
|
|
@@ -164,7 +164,7 @@ export function createChannelReplyTool({
|
|
|
164
164
|
const missingResolveChoice = missingReviewThreadResolveChoiceError({
|
|
165
165
|
origin,
|
|
166
166
|
text,
|
|
167
|
-
|
|
167
|
+
moreWorkThisTurn: keepTurnAlive,
|
|
168
168
|
resolveReviewThread: params.resolve_review_thread,
|
|
169
169
|
})
|
|
170
170
|
if (missingResolveChoice) {
|
|
@@ -185,7 +185,7 @@ export function createChannelReplyTool({
|
|
|
185
185
|
chat: origin.chat,
|
|
186
186
|
thread: origin.thread,
|
|
187
187
|
text,
|
|
188
|
-
|
|
188
|
+
moreWorkThisTurn: keepTurnAlive,
|
|
189
189
|
resolveReviewThread: params.resolve_review_thread === true,
|
|
190
190
|
})
|
|
191
191
|
if (falseReceipt.kind === 'block') {
|
|
@@ -208,7 +208,7 @@ export function createChannelReplyTool({
|
|
|
208
208
|
thread: origin.thread,
|
|
209
209
|
text,
|
|
210
210
|
wantsResolve: params.resolve_review_thread === true,
|
|
211
|
-
|
|
211
|
+
moreWorkThisTurn: keepTurnAlive,
|
|
212
212
|
getReviewState: (req) => router.getReviewState(req),
|
|
213
213
|
})
|
|
214
214
|
if (rereview.block) {
|
|
@@ -259,19 +259,19 @@ export function createChannelReplyTool({
|
|
|
259
259
|
),
|
|
260
260
|
)
|
|
261
261
|
}
|
|
262
|
-
// `
|
|
262
|
+
// `more_work_this_turn` is read by the router's terminal hook (installChannelReplyTerminalHook),
|
|
263
263
|
// not by this tool — it suppresses the post-reply abort so a multi-step turn
|
|
264
264
|
// keeps going. Success-only: a denied reply never ran, so there is no turn to keep.
|
|
265
265
|
const details: {
|
|
266
266
|
ok: boolean
|
|
267
267
|
error?: string
|
|
268
|
-
|
|
268
|
+
more_work_this_turn?: boolean
|
|
269
269
|
messageId?: string
|
|
270
270
|
messageIds?: readonly string[]
|
|
271
271
|
} = result.ok
|
|
272
272
|
? {
|
|
273
273
|
ok: true,
|
|
274
|
-
...(keepTurnAlive ? {
|
|
274
|
+
...(keepTurnAlive ? { more_work_this_turn: true } : {}),
|
|
275
275
|
...(result.messageId !== undefined ? { messageId: result.messageId } : {}),
|
|
276
276
|
...(result.messageIds !== undefined ? { messageIds: result.messageIds } : {}),
|
|
277
277
|
}
|
|
@@ -329,13 +329,13 @@ export function createChannelReplyTool({
|
|
|
329
329
|
function missingReviewThreadResolveChoiceError(input: {
|
|
330
330
|
origin: ChannelReplyOrigin
|
|
331
331
|
text: string | undefined
|
|
332
|
-
|
|
332
|
+
moreWorkThisTurn: boolean
|
|
333
333
|
resolveReviewThread: boolean | undefined
|
|
334
334
|
}): string {
|
|
335
335
|
const isGithubPrReviewThread =
|
|
336
336
|
input.origin.adapter === 'github' && /^pr:\d+$/.test(input.origin.chat) && input.origin.thread !== null
|
|
337
337
|
const hasText = input.text !== undefined && input.text.trim() !== ''
|
|
338
|
-
if (!isGithubPrReviewThread || !hasText || input.
|
|
338
|
+
if (!isGithubPrReviewThread || !hasText || input.moreWorkThisTurn || input.resolveReviewThread !== undefined) {
|
|
339
339
|
return ''
|
|
340
340
|
}
|
|
341
341
|
return (
|
|
@@ -185,7 +185,7 @@ export function createChannelSendTool({
|
|
|
185
185
|
// via channel_send, and the model kept acknowledging "addressed" without
|
|
186
186
|
// the flag, stranding the thread — channel_reply forced the choice but
|
|
187
187
|
// channel_send did not, so the most-used resolve path was the least
|
|
188
|
-
// guarded. A send is always terminal, so there is no
|
|
188
|
+
// guarded. A send is always terminal, so there is no more-work exemption.
|
|
189
189
|
const missingResolveChoice = missingReviewThreadResolveChoiceError({
|
|
190
190
|
adapter,
|
|
191
191
|
chat: params.chat,
|
|
@@ -209,7 +209,7 @@ export function createChannelSendTool({
|
|
|
209
209
|
chat: params.chat,
|
|
210
210
|
thread: params.thread ?? null,
|
|
211
211
|
text: bodyText,
|
|
212
|
-
|
|
212
|
+
moreWorkThisTurn: false,
|
|
213
213
|
resolveReviewThread: wantsResolve,
|
|
214
214
|
})
|
|
215
215
|
if (falseReceipt.kind === 'block') {
|
|
@@ -231,7 +231,7 @@ export function createChannelSendTool({
|
|
|
231
231
|
thread: params.thread ?? null,
|
|
232
232
|
text: bodyText,
|
|
233
233
|
wantsResolve,
|
|
234
|
-
|
|
234
|
+
moreWorkThisTurn: false,
|
|
235
235
|
getReviewState: (req) => router.getReviewState(req),
|
|
236
236
|
})
|
|
237
237
|
if (rereview.block) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { hooklessGitArgs } from '@/git/hookless'
|
|
2
1
|
import { resolveAgentGit } from '@/git/resolve-agent-git'
|
|
2
|
+
import { runGit } from '@/git/run'
|
|
3
3
|
import type { ContentPart, ToolResult } from '@/plugin'
|
|
4
4
|
|
|
5
5
|
export const GUARD_UNCOMMITTED_CHANGES = 'uncommittedChanges'
|
|
@@ -63,16 +63,9 @@ const defaultDeps: UncommittedChangesDeps = {
|
|
|
63
63
|
const bun = getBun()
|
|
64
64
|
if (!bun) return null
|
|
65
65
|
try {
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
stdout: 'pipe',
|
|
70
|
-
stderr: 'pipe',
|
|
71
|
-
})
|
|
72
|
-
const exit = await proc.exited
|
|
73
|
-
if (exit !== 0) return null
|
|
74
|
-
const text = await new Response(proc.stdout).text()
|
|
75
|
-
return parsePorcelain(text)
|
|
66
|
+
const { exitCode, stdout } = await runGit(bun, agentDir, [...gitArgs, 'status', '--porcelain=v1'])
|
|
67
|
+
if (exitCode !== 0) return null
|
|
68
|
+
return parsePorcelain(stdout)
|
|
76
69
|
} catch {
|
|
77
70
|
return null
|
|
78
71
|
}
|
|
@@ -5,9 +5,9 @@ import { basename, join } from 'node:path'
|
|
|
5
5
|
|
|
6
6
|
import { z } from 'zod'
|
|
7
7
|
|
|
8
|
-
import { hooklessGitArgs } from '@/git/hookless'
|
|
9
8
|
import { withGitLock } from '@/git/mutex'
|
|
10
9
|
import { type AgentGit, resolveAgentGit } from '@/git/resolve-agent-git'
|
|
10
|
+
import { runGit } from '@/git/run'
|
|
11
11
|
import { defineTool, lsTool, readTool, type Subagent, writeTool } from '@/plugin'
|
|
12
12
|
import { formatLocalDate, formatLocalDateTime } from '@/shared'
|
|
13
13
|
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
} from './dreaming-state'
|
|
27
27
|
import { parseShard, renderShard, type ShardFrontmatter } from './frontmatter'
|
|
28
28
|
import { listShardSlugs, loadAllShards, loadShard, type TopicShard } from './load-shards'
|
|
29
|
+
import { formatMemorySnapshot } from './memory-snapshot'
|
|
29
30
|
import { referencesDir, streamFilePath, streamsDir, topicShardPath, topicsDir } from './paths'
|
|
30
31
|
import { renderReference } from './references/frontmatter'
|
|
31
32
|
import { loadAllReferences, type Reference } from './references/load-references'
|
|
@@ -773,13 +774,8 @@ async function commitMemorySnapshotUnlocked(cwd: string): Promise<void> {
|
|
|
773
774
|
return
|
|
774
775
|
}
|
|
775
776
|
|
|
776
|
-
const add = bun.
|
|
777
|
-
|
|
778
|
-
cwd,
|
|
779
|
-
stdout: 'pipe',
|
|
780
|
-
stderr: 'pipe',
|
|
781
|
-
})
|
|
782
|
-
if ((await add.exited) !== 0) {
|
|
777
|
+
const add = await runGit(bun, cwd, [...repo.gitArgs, 'add', '-f', '--', ...presentPaths])
|
|
778
|
+
if (add.exitCode !== 0) {
|
|
783
779
|
await applySkipWorktree(bun, cwd, repo)
|
|
784
780
|
return
|
|
785
781
|
}
|
|
@@ -788,21 +784,20 @@ async function commitMemorySnapshotUnlocked(cwd: string): Promise<void> {
|
|
|
788
784
|
// pathspec only references files git knows about. `git commit -- foo bar/`
|
|
789
785
|
// fails outright when `bar/` matches no tracked file, even if `foo` is
|
|
790
786
|
// staged.
|
|
791
|
-
const stagedNames = bun
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
if ((await stagedNames.exited) !== 0) {
|
|
787
|
+
const stagedNames = await runGit(bun, cwd, [
|
|
788
|
+
...repo.gitArgs,
|
|
789
|
+
'diff',
|
|
790
|
+
'--cached',
|
|
791
|
+
'--name-only',
|
|
792
|
+
'-z',
|
|
793
|
+
'--',
|
|
794
|
+
...SNAPSHOT_PATHS,
|
|
795
|
+
])
|
|
796
|
+
if (stagedNames.exitCode !== 0) {
|
|
802
797
|
await applySkipWorktree(bun, cwd, repo)
|
|
803
798
|
return
|
|
804
799
|
}
|
|
805
|
-
const staged =
|
|
800
|
+
const staged = stagedNames.stdout.split('\0').filter((p) => p.length > 0)
|
|
806
801
|
if (staged.length === 0) {
|
|
807
802
|
await applySkipWorktree(bun, cwd, repo)
|
|
808
803
|
return
|
|
@@ -810,13 +805,7 @@ async function commitMemorySnapshotUnlocked(cwd: string): Promise<void> {
|
|
|
810
805
|
|
|
811
806
|
const message = await buildCommitMessage(bun, cwd, staged, undefined, repo.gitArgs)
|
|
812
807
|
|
|
813
|
-
|
|
814
|
-
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'commit', '-m', message, '--only', '--', ...staged])],
|
|
815
|
-
cwd,
|
|
816
|
-
stdout: 'pipe',
|
|
817
|
-
stderr: 'pipe',
|
|
818
|
-
})
|
|
819
|
-
await commit.exited
|
|
808
|
+
await runGit(bun, cwd, [...repo.gitArgs, 'commit', '-m', message, '--only', '--', ...staged])
|
|
820
809
|
|
|
821
810
|
await applySkipWorktree(bun, cwd, repo)
|
|
822
811
|
}
|
|
@@ -867,14 +856,9 @@ async function buildDreamSummary(
|
|
|
867
856
|
): Promise<string> {
|
|
868
857
|
// numstat: `<added>\t<deleted>\t<path>` per line. Use NUL-terminated so paths
|
|
869
858
|
// with whitespace round-trip; -z switches the record separator to NUL.
|
|
870
|
-
const numstat = bun
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
stdout: 'pipe',
|
|
874
|
-
stderr: 'pipe',
|
|
875
|
-
})
|
|
876
|
-
const raw = await new Response(numstat.stdout).text()
|
|
877
|
-
if ((await numstat.exited) !== 0) return 'snapshot'
|
|
859
|
+
const numstat = await runGit(bun, cwd, [...gitArgs, 'diff', '--cached', '--numstat', '-z', '--', ...staged])
|
|
860
|
+
if (numstat.exitCode !== 0) return 'snapshot'
|
|
861
|
+
const raw = numstat.stdout
|
|
878
862
|
|
|
879
863
|
let fragmentLines = 0
|
|
880
864
|
const streamPaths = new Set<string>()
|
|
@@ -924,14 +908,9 @@ async function listNewlyAddedSkills(
|
|
|
924
908
|
staged: string[],
|
|
925
909
|
gitArgs: readonly string[] = [],
|
|
926
910
|
): Promise<string[]> {
|
|
927
|
-
const proc = bun
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
stdout: 'pipe',
|
|
931
|
-
stderr: 'pipe',
|
|
932
|
-
})
|
|
933
|
-
const raw = await new Response(proc.stdout).text()
|
|
934
|
-
if ((await proc.exited) !== 0) return []
|
|
911
|
+
const proc = await runGit(bun, cwd, [...gitArgs, 'diff', '--cached', '--name-status', '-z', '--', ...staged])
|
|
912
|
+
if (proc.exitCode !== 0) return []
|
|
913
|
+
const raw = proc.stdout
|
|
935
914
|
|
|
936
915
|
// `--name-status -z` interleaves status and path as separate NUL records:
|
|
937
916
|
// `A\0path\0M\0other\0...`. Pair them up.
|
|
@@ -952,39 +931,21 @@ async function listTrackedSnapshotFiles(
|
|
|
952
931
|
cwd: string,
|
|
953
932
|
repo: AgentGit,
|
|
954
933
|
): Promise<string[]> {
|
|
955
|
-
const ls = bun.
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
stdout: 'pipe',
|
|
959
|
-
stderr: 'pipe',
|
|
960
|
-
})
|
|
961
|
-
if ((await ls.exited) !== 0) return []
|
|
962
|
-
const raw = await new Response(ls.stdout).text()
|
|
963
|
-
return raw.split('\0').filter((p) => p.length > 0)
|
|
934
|
+
const ls = await runGit(bun, cwd, [...repo.gitArgs, 'ls-files', '-z', '--', ...SNAPSHOT_PATHS])
|
|
935
|
+
if (ls.exitCode !== 0) return []
|
|
936
|
+
return ls.stdout.split('\0').filter((p) => p.length > 0)
|
|
964
937
|
}
|
|
965
938
|
|
|
966
939
|
async function clearSkipWorktree(bun: { spawn: typeof Bun.spawn }, cwd: string, repo: AgentGit): Promise<void> {
|
|
967
940
|
const files = await listTrackedSnapshotFiles(bun, cwd, repo)
|
|
968
941
|
if (files.length === 0) return
|
|
969
|
-
|
|
970
|
-
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'update-index', '--no-skip-worktree', '--', ...files])],
|
|
971
|
-
cwd,
|
|
972
|
-
stdout: 'pipe',
|
|
973
|
-
stderr: 'pipe',
|
|
974
|
-
})
|
|
975
|
-
await proc.exited
|
|
942
|
+
await runGit(bun, cwd, [...repo.gitArgs, 'update-index', '--no-skip-worktree', '--', ...files])
|
|
976
943
|
}
|
|
977
944
|
|
|
978
945
|
async function applySkipWorktree(bun: { spawn: typeof Bun.spawn }, cwd: string, repo: AgentGit): Promise<void> {
|
|
979
946
|
const files = await listTrackedSnapshotFiles(bun, cwd, repo)
|
|
980
947
|
if (files.length === 0) return
|
|
981
|
-
|
|
982
|
-
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'update-index', '--skip-worktree', '--', ...files])],
|
|
983
|
-
cwd,
|
|
984
|
-
stdout: 'pipe',
|
|
985
|
-
stderr: 'pipe',
|
|
986
|
-
})
|
|
987
|
-
await proc.exited
|
|
948
|
+
await runGit(bun, cwd, [...repo.gitArgs, 'update-index', '--skip-worktree', '--', ...files])
|
|
988
949
|
}
|
|
989
950
|
|
|
990
951
|
export const DREAMING_SYSTEM_PROMPT = `You are typeclaw's dreaming subagent.
|
|
@@ -1477,7 +1438,7 @@ export function createDreamingSubagent(options: CreateDreamingSubagentOptions =
|
|
|
1477
1438
|
try {
|
|
1478
1439
|
await commit(ctx.payload.agentDir)
|
|
1479
1440
|
logger.info(
|
|
1480
|
-
`[dreaming] done topics_created=${metrics.topicsCreated} topics_removed=${metrics.topicsRemoved} superseded_new=${metrics.supersededDelta} fragments_dropped=${compaction.fragmentsDropped} over_budget=${overBudget.length} references_demoted=${referencesDemoted} references_evicted=${referencesEvicted} elapsed_ms=${Date.now() - start}`,
|
|
1441
|
+
`[dreaming] done topics_created=${metrics.topicsCreated} topics_removed=${metrics.topicsRemoved} superseded_new=${metrics.supersededDelta} fragments_dropped=${compaction.fragmentsDropped} over_budget=${overBudget.length} references_demoted=${referencesDemoted} references_evicted=${referencesEvicted} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
|
|
1481
1442
|
)
|
|
1482
1443
|
} catch (err) {
|
|
1483
1444
|
const message = err instanceof Error ? err.message : String(err)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readdir, readFile, stat } from 'node:fs/promises'
|
|
1
|
+
import { mkdir, readdir, readFile, stat } from 'node:fs/promises'
|
|
2
2
|
|
|
3
3
|
import { parseShard, type ShardFrontmatter } from './frontmatter'
|
|
4
4
|
import { topicShardPath, topicsDir } from './paths'
|
|
@@ -117,11 +117,23 @@ export async function loadShard(
|
|
|
117
117
|
}
|
|
118
118
|
|
|
119
119
|
export async function listShardSlugs(agentDir: string): Promise<string[]> {
|
|
120
|
+
const dir = topicsDir(agentDir)
|
|
120
121
|
let names: string[]
|
|
121
122
|
try {
|
|
122
|
-
names = await readdir(
|
|
123
|
+
names = await readdir(dir)
|
|
123
124
|
} catch (err) {
|
|
124
|
-
|
|
125
|
+
// ENOENT: the topics dir doesn't exist yet (fresh agent, or a wiped memory
|
|
126
|
+
// tree). Self-heal by recreating it so every downstream reader -- per-turn
|
|
127
|
+
// injection, memory_search, dreaming, the startup index build, and the
|
|
128
|
+
// doctor vector-index check -- sees a real, empty directory instead of a
|
|
129
|
+
// missing one. mkdir is idempotent, so a lost race just no-ops.
|
|
130
|
+
if (isEnoent(err)) return healMissingTopicsDir(dir)
|
|
131
|
+
// ENOTDIR: a bind-mount/tmpfs coherency hiccup can momentarily replace the
|
|
132
|
+
// topics dir (or a parent path component) with a non-directory; readdir
|
|
133
|
+
// then throws ENOTDIR. We can't self-heal this -- mkdir over a regular file
|
|
134
|
+
// would either fail or clobber operator data -- so degrade to empty and let
|
|
135
|
+
// the transient host state resolve itself, exactly as before.
|
|
136
|
+
if (isEnotdir(err)) return []
|
|
125
137
|
throw err
|
|
126
138
|
}
|
|
127
139
|
|
|
@@ -131,6 +143,18 @@ export async function listShardSlugs(agentDir: string): Promise<string[]> {
|
|
|
131
143
|
.sort()
|
|
132
144
|
}
|
|
133
145
|
|
|
146
|
+
async function healMissingTopicsDir(dir: string): Promise<string[]> {
|
|
147
|
+
try {
|
|
148
|
+
await mkdir(dir, { recursive: true })
|
|
149
|
+
} catch (err) {
|
|
150
|
+
// A concurrent healer already created it, or the parent briefly became a
|
|
151
|
+
// non-directory. Either way the empty-list contract still holds; never let
|
|
152
|
+
// the heal attempt turn a self-healing state into a hard error.
|
|
153
|
+
if (!isEnotdir(err)) throw err
|
|
154
|
+
}
|
|
155
|
+
return []
|
|
156
|
+
}
|
|
157
|
+
|
|
134
158
|
// Test-only helper. Clears the in-memory shard cache so tests that exercise
|
|
135
159
|
// the cache invalidation path can simulate a cold start without spinning up a
|
|
136
160
|
// fresh process.
|
|
@@ -180,3 +204,7 @@ function getOrCreateCache(agentDir: string): Map<string, ShardCacheEntry> {
|
|
|
180
204
|
function isEnoent(err: unknown): boolean {
|
|
181
205
|
return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOENT'
|
|
182
206
|
}
|
|
207
|
+
|
|
208
|
+
function isEnotdir(err: unknown): boolean {
|
|
209
|
+
return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOTDIR'
|
|
210
|
+
}
|
|
@@ -6,6 +6,7 @@ import { formatLocalDate } from '@/shared'
|
|
|
6
6
|
|
|
7
7
|
import { advanceWatermarkTool, createAppendTool, type FragmentsAppendedHook } from './append-tool'
|
|
8
8
|
import { findEntryTool } from './find-entry-tool'
|
|
9
|
+
import { formatMemorySnapshot } from './memory-snapshot'
|
|
9
10
|
import { streamFilePath, streamsDir } from './paths'
|
|
10
11
|
import { listReferenceSlugs } from './references/load-references'
|
|
11
12
|
import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
|
|
@@ -352,7 +353,7 @@ export function createMemoryLoggerSubagent(
|
|
|
352
353
|
await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark) })
|
|
353
354
|
const fragmentsWritten = (await countFragments(streamFile)) - fragmentsBefore
|
|
354
355
|
logger.info(
|
|
355
|
-
`[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start}`,
|
|
356
|
+
`[memory-logger] ${ctx.payload.parentSessionId} done fragments_written=${fragmentsWritten} elapsed_ms=${Date.now() - start} ${formatMemorySnapshot()}`,
|
|
356
357
|
)
|
|
357
358
|
} catch (err) {
|
|
358
359
|
const message = err instanceof Error ? err.message : String(err)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type MemorySample = Pick<NodeJS.MemoryUsage, 'rss' | 'heapUsed' | 'heapTotal' | 'external' | 'arrayBuffers'>
|
|
2
|
+
|
|
3
|
+
const BYTES_PER_MB = 1024 * 1024
|
|
4
|
+
|
|
5
|
+
export function formatMemorySnapshot(sample: MemorySample = process.memoryUsage()): string {
|
|
6
|
+
const mb = (bytes: number): number => Math.round(bytes / BYTES_PER_MB)
|
|
7
|
+
return `rss_mb=${mb(sample.rss)} heapUsed_mb=${mb(sample.heapUsed)} external_mb=${mb(sample.external)} arrayBuffers_mb=${mb(sample.arrayBuffers)}`
|
|
8
|
+
}
|
|
@@ -6,6 +6,7 @@ import { join } from 'node:path'
|
|
|
6
6
|
// values are pulled lazily via `loadTransformers()` below.
|
|
7
7
|
import type { env as TransformersEnvValue, pipeline as TransformersPipeline } from '@huggingface/transformers'
|
|
8
8
|
|
|
9
|
+
import { createKeyedSemaphore } from '@/agent/tools/keyed-semaphore'
|
|
9
10
|
import {
|
|
10
11
|
assertModelCacheCompatible,
|
|
11
12
|
EMBEDDING_DIMS,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
import { getResolvedTransformersVersion } from '@/models/transformers-version'
|
|
17
18
|
|
|
18
19
|
import { homeRoot } from '../../../hostd/paths'
|
|
20
|
+
import { formatMemorySnapshot } from '../memory-snapshot'
|
|
19
21
|
import { resolveEmbedThreadCount } from './embed-threads'
|
|
20
22
|
import { type BoundedText, boundEmbeddableText, MAX_MODEL_TOKENS } from './truncation'
|
|
21
23
|
|
|
@@ -36,6 +38,21 @@ export type EmbedType = 'query' | 'passage'
|
|
|
36
38
|
// loaded once and reused across chunks, so the only cost is sequential passes.
|
|
37
39
|
const EMBED_BATCH_SIZE = 64
|
|
38
40
|
|
|
41
|
+
// Concurrent embeds inside inference at once, process-wide. The intra-op thread
|
|
42
|
+
// cap (embed-threads.ts) bounds ONE embed's footprint, but nothing bounded how
|
|
43
|
+
// MANY embeds run at once: a busy channel turn's hybridSearch, a dreaming pass,
|
|
44
|
+
// and a memory-logger append can all be inside the shared ONNX session
|
|
45
|
+
// simultaneously (concurrent `Run()` is allowed on the Node backend), stacking
|
|
46
|
+
// their activation memory into one RSS spike that the OOM killer takes with a
|
|
47
|
+
// bare SIGKILL. This caps that stack. Chosen 2, not 1: the workload is
|
|
48
|
+
// latency-sensitive per-turn query embeds, so a hard serial would park a live
|
|
49
|
+
// turn behind a whole dreaming batch; 2 keeps one lane free while still bounding
|
|
50
|
+
// the worst-case multiplier. Single fixed key — the limit is global, not
|
|
51
|
+
// per-key.
|
|
52
|
+
const EMBED_CONCURRENCY = 2
|
|
53
|
+
const EMBED_SEMAPHORE_KEY = 'embed'
|
|
54
|
+
const embedSemaphore = createKeyedSemaphore({ concurrency: EMBED_CONCURRENCY })
|
|
55
|
+
|
|
39
56
|
type TransformersEnv = typeof TransformersEnvValue
|
|
40
57
|
type FeatureExtractor = Awaited<ReturnType<typeof TransformersPipeline<'feature-extraction'>>>
|
|
41
58
|
|
|
@@ -154,14 +171,16 @@ export class Embedder {
|
|
|
154
171
|
// upserts) finish in a pass or two and would only spam the logs.
|
|
155
172
|
const reportProgress = prefixed.length >= LARGE_EMBED
|
|
156
173
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
174
|
+
return embedSemaphore.run(EMBED_SEMAPHORE_KEY, async () => {
|
|
175
|
+
const embeddings: Float32Array[] = []
|
|
176
|
+
for (let start = 0; start < prefixed.length; start += EMBED_BATCH_SIZE) {
|
|
177
|
+
const batch = prefixed.slice(start, start + EMBED_BATCH_SIZE)
|
|
178
|
+
const output = await this.extractor(batch, { pooling: 'mean', normalize: true })
|
|
179
|
+
embeddings.push(...toEmbeddings(output.data, batch.length))
|
|
180
|
+
if (reportProgress) logEmbedProgress(embeddings.length, prefixed.length, type)
|
|
181
|
+
}
|
|
182
|
+
return embeddings
|
|
183
|
+
})
|
|
165
184
|
}
|
|
166
185
|
}
|
|
167
186
|
|
|
@@ -213,7 +232,7 @@ function warnIfBounded(results: readonly BoundedText[], type: EmbedType): void {
|
|
|
213
232
|
const LARGE_EMBED = 256
|
|
214
233
|
|
|
215
234
|
function logEmbedBatch(count: number, type: EmbedType): void {
|
|
216
|
-
const line = `[memory] vector embedding: ${count} ${type} input(s) (chunked at ${EMBED_BATCH_SIZE}/pass)`
|
|
235
|
+
const line = `[memory] vector embedding: ${count} ${type} input(s) (chunked at ${EMBED_BATCH_SIZE}/pass) ${formatMemorySnapshot()}`
|
|
217
236
|
if (count >= LARGE_EMBED) {
|
|
218
237
|
console.info(`${line} — large build, this may take a while`)
|
|
219
238
|
} else {
|