typeclaw 0.46.0 → 0.46.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -2
- package/src/agent/multimodal/looker.ts +11 -1
- package/src/agent/network/safe-http.ts +17 -8
- package/src/agent/plugin-tools.ts +1 -3
- package/src/agent/tool-file-safety.ts +396 -136
- package/src/agent/tools/webfetch/fetch.ts +22 -2
- package/src/agent/tools-without-local-file-operands.ts +9 -6
- package/src/bundled-plugins/backup/README.md +23 -10
- package/src/bundled-plugins/backup/git-auth.ts +6 -4
- package/src/bundled-plugins/backup/index.ts +83 -43
- package/src/bundled-plugins/backup/runner.ts +83 -0
- package/src/bundled-plugins/github-cli-auth/git-askpass.ts +74 -26
- package/src/bundled-plugins/github-cli-auth/git-command.ts +224 -36
- package/src/bundled-plugins/github-cli-auth/graphql-auth-nudge.ts +9 -9
- package/src/bundled-plugins/github-cli-auth/index.ts +110 -31
- package/src/bundled-plugins/github-cli-auth/token-class.ts +4 -6
- package/src/bundled-plugins/memory/README.md +9 -0
- package/src/bundled-plugins/memory/vector/embed-threads.ts +36 -0
- package/src/bundled-plugins/memory/vector/embedder.ts +19 -1
- package/src/bundled-plugins/researcher/write-report.ts +3 -3
- package/src/bundled-plugins/security/policies/private-surface-read.ts +610 -50
- package/src/bundled-plugins/security/policies/ssrf.ts +119 -34
- package/src/channels/adapters/github/inbound.ts +11 -0
- package/src/channels/adapters/github/index.ts +48 -54
- package/src/channels/adapters/github/reconcile-cooldown-store.ts +178 -0
- package/src/channels/adapters/github/reconcile-open-prs.ts +31 -0
- package/src/channels/adapters/slack-bot-edit.ts +85 -20
- package/src/channels/adapters/slack-bot.ts +7 -2
- package/src/channels/adapters/slack-format.ts +245 -0
- package/src/channels/github-token-bridge.ts +2 -3
- package/src/container/docker-app.ts +42 -13
- package/src/container/start.ts +13 -0
- package/src/cron/bridge.ts +9 -2
- package/src/doctor/checks.ts +5 -2
- package/src/git/bound-gc-config.ts +82 -0
- package/src/hostd/spawn.ts +133 -47
- package/src/init/dockerfile.ts +13 -0
- package/src/inspect/replay.ts +21 -9
- package/src/inspect/session-list.ts +71 -33
- package/src/mcp/tools.ts +1 -0
- package/src/network/internal-destinations.ts +267 -0
- package/src/plugin/types.ts +3 -2
- package/src/sandbox/env-exposure.ts +18 -15
- package/src/sandbox/policy.ts +5 -3
- package/src/sandbox/privileged-runtime.ts +1 -0
- package/src/skills/typeclaw-config/SKILL.md +22 -6
- package/src/skills/typeclaw-permissions/SKILL.md +22 -9
- package/src/skills/typeclaw-plugins/SKILL.md +37 -0
- package/src/usage/scan.ts +11 -2
- package/typeclaw.schema.json +5 -0
- package/src/git/secret-history.ts +0 -371
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
type PublicHttpDependencies,
|
|
8
8
|
type PublicHttpResponse,
|
|
9
9
|
} from '@/agent/network/safe-http'
|
|
10
|
+
import type { InternalDestinationPolicy } from '@/network/internal-destinations'
|
|
10
11
|
|
|
11
12
|
import { MAX_RESPONSE_BYTES } from './types'
|
|
12
13
|
|
|
@@ -69,6 +70,7 @@ export async function fetchWithLimits(
|
|
|
69
70
|
parentSignal?: AbortSignal,
|
|
70
71
|
antibotWarmup: AntibotWarmup = 'auto',
|
|
71
72
|
network: WebFetchNetworkDependencies = forceFallbackForTest ? testFetchDependencies : defaultPublicHttpDependencies,
|
|
73
|
+
internalDestinationPolicy?: InternalDestinationPolicy,
|
|
72
74
|
): Promise<FetchResult> {
|
|
73
75
|
const controller = new AbortController()
|
|
74
76
|
const timeout = setTimeout(() => controller.abort(new Error('timeout')), timeoutSeconds * 1000)
|
|
@@ -78,7 +80,15 @@ export async function fetchWithLimits(
|
|
|
78
80
|
try {
|
|
79
81
|
const cookieJar = new WarmupCookieJar()
|
|
80
82
|
const visitedUrls = new Set<string>()
|
|
81
|
-
const first = await fetchOnce(
|
|
83
|
+
const first = await fetchOnce(
|
|
84
|
+
url,
|
|
85
|
+
controller.signal,
|
|
86
|
+
network,
|
|
87
|
+
REQUEST_HEADERS,
|
|
88
|
+
cookieJar,
|
|
89
|
+
visitedUrls,
|
|
90
|
+
internalDestinationPolicy,
|
|
91
|
+
)
|
|
82
92
|
const cookieNames = cookieJar.names()
|
|
83
93
|
const warmup =
|
|
84
94
|
antibotWarmup === 'auto' &&
|
|
@@ -97,7 +107,15 @@ export async function fetchWithLimits(
|
|
|
97
107
|
initialSetCookieNames: cookieNames,
|
|
98
108
|
})
|
|
99
109
|
}
|
|
100
|
-
const replay = await fetchOnce(
|
|
110
|
+
const replay = await fetchOnce(
|
|
111
|
+
url,
|
|
112
|
+
controller.signal,
|
|
113
|
+
network,
|
|
114
|
+
REQUEST_HEADERS,
|
|
115
|
+
cookieJar,
|
|
116
|
+
undefined,
|
|
117
|
+
internalDestinationPolicy,
|
|
118
|
+
)
|
|
101
119
|
return toFetchResult(replay, {
|
|
102
120
|
attempted: true,
|
|
103
121
|
triggered: true,
|
|
@@ -152,11 +170,13 @@ async function fetchOnce(
|
|
|
152
170
|
headers: Record<string, string>,
|
|
153
171
|
cookieJar?: WarmupCookieJar,
|
|
154
172
|
visitedUrls?: Set<string>,
|
|
173
|
+
internalDestinationPolicy?: InternalDestinationPolicy,
|
|
155
174
|
): Promise<BufferedResponse> {
|
|
156
175
|
const { response, finalUrl } = await requestPublicHttpUrl(url, {
|
|
157
176
|
signal,
|
|
158
177
|
headers,
|
|
159
178
|
dependencies: network,
|
|
179
|
+
...(internalDestinationPolicy === undefined ? {} : { internalDestinationPolicy }),
|
|
160
180
|
headersForUrl(currentUrl): Record<string, string> {
|
|
161
181
|
visitedUrls?.add(currentUrl)
|
|
162
182
|
const cookie = cookieJar?.header(currentUrl)
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
// First-party system tools
|
|
2
|
-
//
|
|
3
|
-
// ids,
|
|
4
|
-
//
|
|
1
|
+
// First-party system tools with no caller-supplied local filesystem operand.
|
|
2
|
+
// Their arguments are control tokens, remote identifiers (channel/workspace/
|
|
3
|
+
// message/thread/cursor ids, subagent names, task ids, role capabilities,
|
|
4
|
+
// reload/stream scopes), or sanitized output metadata — none is dereferenced as
|
|
5
|
+
// a caller-selected local path. Listing the whole tool is more robust than enumerating each
|
|
5
6
|
// operand: these schemas gain new id/enum fields often, and every such field
|
|
6
|
-
// hits the same
|
|
7
|
-
//
|
|
7
|
+
// hits the same conservative inference rules (a Slack `ts` like
|
|
8
|
+
// "1699999999.000100" looks like a dotted filename; a value equal to an agent-root dir like "memory" hits the
|
|
8
9
|
// fs-existence probe; a base64 cursor carries a "/"). The security cost is zero
|
|
9
10
|
// TODAY because none reads a local file; the risk is future drift, so this set
|
|
10
11
|
// is the single fence, shared by BOTH enforcement points — the file-operand
|
|
@@ -17,7 +18,9 @@ export const TOOLS_WITHOUT_LOCAL_FILE_OPERANDS: ReadonlySet<string> = new Set([
|
|
|
17
18
|
'channel_history',
|
|
18
19
|
'channel_edit',
|
|
19
20
|
'channel_react',
|
|
21
|
+
'channel_fetch_attachment',
|
|
20
22
|
'look_at_channel_attachment',
|
|
23
|
+
'post_github_review',
|
|
21
24
|
'stream_snapshot',
|
|
22
25
|
'grant_role',
|
|
23
26
|
'spawn_subagent',
|
|
@@ -11,18 +11,20 @@ This plugin is **auto-loaded** by every TypeClaw agent. There is no `plugins[]`
|
|
|
11
11
|
"backup": {
|
|
12
12
|
"enabled": true,
|
|
13
13
|
"idleMs": 30000,
|
|
14
|
-
"pushToOrigin": true
|
|
14
|
+
"pushToOrigin": true,
|
|
15
|
+
"maintenance": true
|
|
15
16
|
}
|
|
16
17
|
}
|
|
17
18
|
```
|
|
18
19
|
|
|
19
|
-
| Field | Default | Effect
|
|
20
|
-
| ------------------------- | ------- |
|
|
21
|
-
| `backup.enabled` | `true` | Master switch. When `false`, all hooks no-op and the runner subagent is never spawned.
|
|
22
|
-
| `backup.idleMs` | `30000` | Debounce window after the agent goes idle (no in-flight prompt turns) before the backup runner fires. Resets on every new prompt. Minimum `1000`.
|
|
23
|
-
| `backup.pushToOrigin` | `true` | When `true`, after committing, the runner attempts `git push`. On non-fast-forward, it `git fetch && git rebase` then re-pushes. On rebase conflict, it aborts the rebase and asks the diagnose subagent to write a human-readable report. Set `false` to commit-only (useful for offline workflows or repos without a remote).
|
|
24
|
-
| `backup.
|
|
25
|
-
| `backup.
|
|
20
|
+
| Field | Default | Effect |
|
|
21
|
+
| ------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
22
|
+
| `backup.enabled` | `true` | Master switch. When `false`, all hooks no-op and the runner subagent is never spawned. |
|
|
23
|
+
| `backup.idleMs` | `30000` | Debounce window after the agent goes idle (no in-flight prompt turns) before the backup runner fires. Resets on every new prompt. Minimum `1000`. |
|
|
24
|
+
| `backup.pushToOrigin` | `true` | When `true`, after committing, the runner attempts `git push`. On non-fast-forward, it `git fetch && git rebase` then re-pushes. On rebase conflict, it aborts the rebase and asks the diagnose subagent to write a human-readable report. Set `false` to commit-only (useful for offline workflows or repos without a remote). |
|
|
25
|
+
| `backup.maintenance` | `true` | When `true`, after a backup that made a local commit, the runner runs a **bounded, best-effort** `git maintenance run` (under the same git lock) with `loose-objects` and/or `incremental-repack` selected independently by loose-object and pack counts. Reclaims what the backup keeps adding — without which loose objects (and later packs) grow until the (now-disabled) auto-gc would fire a memory-heavy full repack. `incremental-repack` (multi-pack-index) leaves the large established pack alone, and both tasks run with `pack.threads=1`/`pack.windowMemory=64m`/`pack.deltaCacheSize=1` so RSS stays bounded. A maintenance failure is logged, never folded into the backup result. Set `false` to disable. |
|
|
26
|
+
| `backup.commitTimeoutMs` | `30000` | Per-command wall clock for local git operations (status/add/commit/diff). Mostly an escape hatch — defaults are generous. |
|
|
27
|
+
| `backup.networkTimeoutMs` | `60000` | Per-command wall clock for network git operations (push/fetch/rebase). Bounds the failure mode where a stuck remote would otherwise hang the runner indefinitely. `GIT_TERMINAL_PROMPT=0` is also set so auth failures fail fast instead of prompting. |
|
|
26
28
|
|
|
27
29
|
All fields are **restart-required** — the plugin reads them once at boot.
|
|
28
30
|
|
|
@@ -57,6 +59,17 @@ On non-fast-forward rejection (either push shape), the runner runs `git fetch` t
|
|
|
57
59
|
|
|
58
60
|
If any network step fails (rebase conflict, auth failure, network timeout), the runner aborts cleanly and spawns the `backup-diagnose` subagent. That subagent has `bash`, `read`, and `write` tools and writes a short human-readable report to `<agentDir>/sessions/backup-diagnostics.log`. The diagnose subagent is explicitly forbidden from force-pushing or resolving merge conflicts itself.
|
|
59
61
|
|
|
62
|
+
## What it maintains
|
|
63
|
+
|
|
64
|
+
Because the backup force-commits `sessions/` every idle window, each cycle lands new objects as small packs. The agent repo disables git's auto-gc (machine-local config written by `typeclaw start`) so a full repack can never fire opportunistically mid-turn — on a GB-scale agent pack that repack was measured at ~1.8GB RSS and OOM-killed the container. Reclamation therefore moves here, to the **idle backup path**:
|
|
65
|
+
|
|
66
|
+
- After a backup that made a **local commit** — including one whose later push failed, since the commit still grew local history and a remote outage must not suppress reclamation — and only when `backup.maintenance` is on, the runner probes `git count-objects -v`. The two reclamation tasks are gated **independently**, because with auto-gc off a commit produces _loose objects_, not packs:
|
|
67
|
+
- `loose-objects` runs when the **loose-object count** (`count:`) exceeds its threshold — the counter that actually grows every idle cycle.
|
|
68
|
+
- `incremental-repack` runs when the **pack count** (`packs:`) exceeds its threshold — consolidating the packs that `loose-objects` itself produces.
|
|
69
|
+
Gating both on pack count alone would let loose objects accumulate forever while packs stayed ~0.
|
|
70
|
+
- **Bounded by construction.** `incremental-repack` uses the multi-pack-index to consolidate only _small_ packs, leaving the large established pack untouched — so both memory and time stay bounded, unlike a monolithic `git gc`. Both tasks are pinned with `-c pack.threads=1 -c pack.windowMemory=64m -c pack.deltaCacheSize=1` so RSS holds even on a repo predating the start-time config.
|
|
71
|
+
- **Same lock, best-effort.** Maintenance runs inside the same `withGitLock` as the commit/push, so a concurrent backup can't race the pack rewrite. Its outcome is logged; a failure is **never** folded into the backup result. Each task consolidates its own input, so the counts reset and both self-throttle — no wall-clock timer.
|
|
72
|
+
|
|
60
73
|
## What it contributes
|
|
61
74
|
|
|
62
75
|
| Kind | Name | Notes |
|
|
@@ -85,6 +98,6 @@ This feature came up as: "periodically check for dirty files and commit; LLM pic
|
|
|
85
98
|
|
|
86
99
|
## Tests
|
|
87
100
|
|
|
88
|
-
- `runner.test.ts` — deterministic runner unit tests (status parsing, force-add of `sessions/`, push-with-upstream, push-and-set-upstream when origin exists but tracking is absent, commit-only on no-origin / detached-HEAD, rebase-on-non-fast-forward for both push shapes, diagnose-on-rebase-conflict, advisory-throw isolation, sanitize-commit-message).
|
|
101
|
+
- `runner.test.ts` — deterministic runner unit tests (status parsing, force-add of `sessions/`, push-with-upstream, push-and-set-upstream when origin exists but tracking is absent, commit-only on no-origin / detached-HEAD, rebase-on-non-fast-forward for both push shapes, diagnose-on-rebase-conflict, advisory-throw isolation, sanitize-commit-message; and `runMaintenance` — independent loose/pack gates, loose-only vs repack-only vs both, bounded `-c` flags, `.gitstore` layout, no-op on non-repo, failure surfaced without throwing, skip-on-probe-failure, plus a **real-Git lifecycle** test proving auto-gc-disabled commits trip the loose gate and a real `git maintenance run` packs them into a valid repo).
|
|
89
102
|
- `git-auth.test.ts` — credential-env resolution (App-auth mints for the origin slug; PAT/SSH/non-github/unavailable-token all fall back to inherited env).
|
|
90
|
-
- `index.test.ts` — plugin composition tests (subagent/hook surface, config schema defaults and validation, debounce, active-turn gating, self-induced-turn exclusion, coalescing).
|
|
103
|
+
- `index.test.ts` — plugin composition tests (subagent/hook surface, config schema defaults and validation, debounce, active-turn gating, self-induced-turn exclusion, coalescing) and `backupCommitted` per-result-kind classification (post-commit push/rebase failures still gate maintenance; `clean`/`no-repo`/`commit-failed`/`aborted` do not).
|
|
@@ -17,14 +17,16 @@ export type BackupPushAuthDeps = {
|
|
|
17
17
|
// The backup runner spawns git directly (not via the bash tool), so the
|
|
18
18
|
// `github-cli-auth` plugin's `tool.before` credential injection never fires for
|
|
19
19
|
// its push. Without this, App-auth agents push with no credentials and fail.
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
20
|
+
// A live App resolver is authoritative even when the process also carries an
|
|
21
|
+
// operator PAT: runtime backup always uses the repo-scoped App credential.
|
|
22
|
+
// Without a resolver, PAT/SSH/credential-helper setups keep using the runner's
|
|
23
|
+
// inherited process env.
|
|
23
24
|
export async function resolveBackupPushAuthEnv(
|
|
24
25
|
cwd: string,
|
|
25
26
|
deps: BackupPushAuthDeps,
|
|
26
27
|
): Promise<BackupGitAuthEnv | null> {
|
|
27
|
-
|
|
28
|
+
const hasAppTokenResolver = deps.hasAppTokenResolver()
|
|
29
|
+
if (!hasAppTokenResolver && !shouldMintAppToken(deps.ghToken, false)) return null
|
|
28
30
|
|
|
29
31
|
const originUrl = await deps.resolveOriginPushUrl(cwd)
|
|
30
32
|
if (originUrl === null) return null
|
|
@@ -6,7 +6,14 @@ import { resolveAgentGit } from '@/git/resolve-agent-git'
|
|
|
6
6
|
import { definePlugin, type PluginContext, type SpawnSubagentOptions, type Subagent } from '@/plugin'
|
|
7
7
|
|
|
8
8
|
import { type BackupPushAuthDeps, makeDefaultAskPassEnsurer, resolveBackupPushAuthEnv } from './git-auth'
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
COMMIT_TIMEOUT_MS,
|
|
11
|
+
makeDefaultGitSpawn,
|
|
12
|
+
NETWORK_TIMEOUT_MS,
|
|
13
|
+
runBackup,
|
|
14
|
+
runMaintenance,
|
|
15
|
+
type BackupResult,
|
|
16
|
+
} from './runner'
|
|
10
17
|
import {
|
|
11
18
|
cleanupMessageFile,
|
|
12
19
|
type CommitMessagePayload,
|
|
@@ -36,6 +43,7 @@ const backupConfigSchema = z
|
|
|
36
43
|
enabled: z.boolean().default(true),
|
|
37
44
|
idleMs: z.number().int().min(MIN_IDLE_MS).default(DEFAULT_IDLE_MS),
|
|
38
45
|
pushToOrigin: z.boolean().default(true),
|
|
46
|
+
maintenance: z.boolean().default(true),
|
|
39
47
|
commitTimeoutMs: z.number().int().min(1).default(COMMIT_TIMEOUT_MS),
|
|
40
48
|
networkTimeoutMs: z.number().int().min(1).default(NETWORK_TIMEOUT_MS),
|
|
41
49
|
})
|
|
@@ -43,6 +51,7 @@ const backupConfigSchema = z
|
|
|
43
51
|
enabled: true,
|
|
44
52
|
idleMs: DEFAULT_IDLE_MS,
|
|
45
53
|
pushToOrigin: true,
|
|
54
|
+
maintenance: true,
|
|
46
55
|
commitTimeoutMs: COMMIT_TIMEOUT_MS,
|
|
47
56
|
networkTimeoutMs: NETWORK_TIMEOUT_MS,
|
|
48
57
|
})
|
|
@@ -50,6 +59,7 @@ const backupConfigSchema = z
|
|
|
50
59
|
const runnerPayloadSchema = z.object({
|
|
51
60
|
agentDir: z.string(),
|
|
52
61
|
pushToOrigin: z.boolean(),
|
|
62
|
+
maintenance: z.boolean(),
|
|
53
63
|
})
|
|
54
64
|
|
|
55
65
|
type RunnerPayload = z.infer<typeof runnerPayloadSchema>
|
|
@@ -60,6 +70,7 @@ export default definePlugin({
|
|
|
60
70
|
const enabled = ctx.config.enabled
|
|
61
71
|
const idleMs = ctx.config.idleMs
|
|
62
72
|
const pushToOrigin = ctx.config.pushToOrigin
|
|
73
|
+
const maintenance = ctx.config.maintenance
|
|
63
74
|
|
|
64
75
|
const activeTurns = new Set<string>()
|
|
65
76
|
let idleTimer: ReturnType<typeof setTimeout> | null = null
|
|
@@ -87,6 +98,7 @@ export default definePlugin({
|
|
|
87
98
|
{
|
|
88
99
|
agentDir: ctx.agentDir,
|
|
89
100
|
pushToOrigin,
|
|
101
|
+
maintenance,
|
|
90
102
|
} satisfies RunnerPayload,
|
|
91
103
|
// The backup runner is a system-level operation that commits +
|
|
92
104
|
// pushes on the operator's behalf. It runs after every idle
|
|
@@ -188,53 +200,81 @@ async function runBackupOnce(
|
|
|
188
200
|
? ((await resolveBackupAuthEnv(payload.agentDir, ctx.github, ctx.logger)) ?? undefined)
|
|
189
201
|
: undefined
|
|
190
202
|
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
|
|
203
|
+
const gitSpawn = makeDefaultGitSpawn()
|
|
204
|
+
const backupDeps = {
|
|
205
|
+
gitSpawn,
|
|
206
|
+
pushEnv,
|
|
207
|
+
pickCommitMessage: async ({ status, diffstat }: { status: string; diffstat: string }): Promise<string> => {
|
|
208
|
+
await cleanupMessageFile(messagePath)
|
|
209
|
+
const messagePayload: CommitMessagePayload = {
|
|
210
|
+
agentDir: payload.agentDir,
|
|
211
|
+
status,
|
|
212
|
+
diffstat,
|
|
213
|
+
outputPath: messagePath,
|
|
214
|
+
}
|
|
215
|
+
try {
|
|
216
|
+
await ctx.spawnSubagent(SUBAGENT_COMMIT_MESSAGE, messagePayload, inheritOwner)
|
|
217
|
+
} catch (err) {
|
|
218
|
+
ctx.logger.warn(
|
|
219
|
+
`${SUBAGENT_COMMIT_MESSAGE} subagent failed, using fallback: ${err instanceof Error ? err.message : String(err)}`,
|
|
220
|
+
)
|
|
221
|
+
}
|
|
222
|
+
const written = await readMessageFile(messagePath)
|
|
223
|
+
await cleanupMessageFile(messagePath)
|
|
224
|
+
return written ?? 'chore: backup'
|
|
225
|
+
},
|
|
226
|
+
diagnoseFailure: async (input: {
|
|
227
|
+
cwd: string
|
|
228
|
+
stage: 'push' | 'rebase'
|
|
229
|
+
exitCode: number
|
|
230
|
+
stderr: string
|
|
231
|
+
stdout: string
|
|
232
|
+
}): Promise<void> => {
|
|
233
|
+
const diagPayload: DiagnoseFailurePayload = {
|
|
234
|
+
agentDir: input.cwd,
|
|
235
|
+
stage: input.stage,
|
|
236
|
+
exitCode: input.exitCode,
|
|
237
|
+
stderr: input.stderr,
|
|
238
|
+
stdout: input.stdout,
|
|
239
|
+
}
|
|
240
|
+
try {
|
|
241
|
+
await ctx.spawnSubagent(SUBAGENT_DIAGNOSE, diagPayload, inheritOwner)
|
|
242
|
+
} catch (err) {
|
|
243
|
+
ctx.logger.warn(`${SUBAGENT_DIAGNOSE} subagent failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
244
|
+
}
|
|
245
|
+
},
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Maintenance runs UNDER THE SAME LOCK as the backup, right after it, so a
|
|
249
|
+
// concurrent backup can't race the repack's index/pack rewrite. It fires
|
|
250
|
+
// whenever a LOCAL COMMIT was made — including when the later push failed, since
|
|
251
|
+
// the loose objects the commit created still need reclaiming (a prolonged remote
|
|
252
|
+
// outage must not suppress it). Best-effort: its outcome is logged, never folded
|
|
253
|
+
// into the returned BackupResult.
|
|
254
|
+
const result = await withGitLock(payload.agentDir, async () => {
|
|
255
|
+
const backup = await runBackup({ cwd: payload.agentDir, pushToOrigin: payload.pushToOrigin }, backupDeps)
|
|
256
|
+
if (payload.maintenance && backupCommitted(backup)) {
|
|
257
|
+
const maint = await runMaintenance(payload.agentDir, backupDeps)
|
|
258
|
+
if (!maint.ok) ctx.logger.warn(`[backup] maintenance failed: ${maint.reason}`)
|
|
259
|
+
else if (maint.kind === 'ran') ctx.logger.info(`[backup] maintenance ran (${(maint.tasks ?? []).join(', ')})`)
|
|
260
|
+
}
|
|
261
|
+
return backup
|
|
262
|
+
})
|
|
233
263
|
|
|
234
264
|
await cleanupMessageFile(messagePath)
|
|
235
265
|
return result
|
|
236
266
|
}
|
|
237
267
|
|
|
268
|
+
// True when a local commit was created this run, regardless of the push outcome.
|
|
269
|
+
// The commit is made before push recovery, so `push-failed`/`rebase-failed` still
|
|
270
|
+
// grew local history and must trigger reclamation; only `clean`/`no-repo` (no
|
|
271
|
+
// commit) and `commit-failed`/`aborted` (commit never happened) skip it. Exported
|
|
272
|
+
// for direct unit tests of this exact per-kind classification.
|
|
273
|
+
export function backupCommitted(result: BackupResult): boolean {
|
|
274
|
+
if (result.ok) return result.kind !== 'clean' && result.kind !== 'no-repo'
|
|
275
|
+
return result.kind === 'push-failed' || result.kind === 'rebase-failed'
|
|
276
|
+
}
|
|
277
|
+
|
|
238
278
|
async function resolveBackupAuthEnv(
|
|
239
279
|
agentDir: string,
|
|
240
280
|
github: PluginContext['github'],
|
|
@@ -6,6 +6,31 @@ import { type AgentGit, resolveAgentGit } from '@/git/resolve-agent-git'
|
|
|
6
6
|
|
|
7
7
|
export const COMMIT_TIMEOUT_MS = 30_000
|
|
8
8
|
export const NETWORK_TIMEOUT_MS = 60_000
|
|
9
|
+
export const MAINTENANCE_TIMEOUT_MS = 120_000
|
|
10
|
+
|
|
11
|
+
// The two maintenance tasks are gated INDEPENDENTLY because a backup commit with
|
|
12
|
+
// auto-gc disabled (Fix 1) produces LOOSE OBJECTS, not packs:
|
|
13
|
+
// - loose-objects fires on the loose-object count (`count:` in count-objects),
|
|
14
|
+
// which is what actually grows every idle backup cycle. Gating this on pack
|
|
15
|
+
// count would let loose objects accumulate forever while packs stayed ~0.
|
|
16
|
+
// - incremental-repack fires on the pack count (`packs:`), consolidating the
|
|
17
|
+
// packs that loose-objects itself produces once they build up.
|
|
18
|
+
// Each task consolidates its own input, so the counts reset and both self-throttle
|
|
19
|
+
// — no wall-clock timer needed.
|
|
20
|
+
const MAINTENANCE_LOOSE_THRESHOLD = 300
|
|
21
|
+
const MAINTENANCE_PACK_THRESHOLD = 12
|
|
22
|
+
|
|
23
|
+
// Memory caps for the maintenance repack, matching the machine-local config Fix 1
|
|
24
|
+
// writes so the bound holds even on a repo that predates it. Passed as `-c` so
|
|
25
|
+
// they apply to this invocation regardless of the persisted config.
|
|
26
|
+
const BOUNDED_PACK_FLAGS = [
|
|
27
|
+
'-c',
|
|
28
|
+
'pack.threads=1',
|
|
29
|
+
'-c',
|
|
30
|
+
'pack.windowMemory=64m',
|
|
31
|
+
'-c',
|
|
32
|
+
'pack.deltaCacheSize=1',
|
|
33
|
+
] as const
|
|
9
34
|
|
|
10
35
|
const RUNTIME_OWNED_PREFIXES = ['memory/'] as const
|
|
11
36
|
const FORCE_ADD_PREFIXES = ['sessions/', 'todo/'] as const
|
|
@@ -152,6 +177,64 @@ export async function runBackup(options: BackupRunnerOptions, deps: BackupRunner
|
|
|
152
177
|
return pushWithRecovery(cwd, deps, repo, plan)
|
|
153
178
|
}
|
|
154
179
|
|
|
180
|
+
export type MaintenanceResult =
|
|
181
|
+
| { ok: true; kind: 'no-repo' | 'skipped' | 'ran'; tasks?: readonly string[] }
|
|
182
|
+
| { ok: false; kind: 'failed'; reason: string }
|
|
183
|
+
|
|
184
|
+
// Bounded, best-effort git maintenance run on the idle backup path. With auto-gc
|
|
185
|
+
// disabled (Fix 1), nothing reclaims what the backup keeps adding, so this runs
|
|
186
|
+
// the two reclamation tasks — each gated on ITS OWN growing counter (see the
|
|
187
|
+
// threshold comment above): `loose-objects` when loose objects pile up (the
|
|
188
|
+
// steady-state effect of every commit), `incremental-repack` when packs pile up.
|
|
189
|
+
// incremental-repack uses the multi-pack-index, touching only SMALL packs and
|
|
190
|
+
// leaving the big established pack alone, so memory and time stay bounded unlike a
|
|
191
|
+
// full `git gc`. Both tasks run under BOUNDED_PACK_FLAGS. NEVER throws: a
|
|
192
|
+
// maintenance failure must not turn a good commit into a reported backup failure.
|
|
193
|
+
// The caller runs this after a commit, holding the same git lock the commit did.
|
|
194
|
+
export type MaintenanceThresholds = { loose?: number; packs?: number }
|
|
195
|
+
|
|
196
|
+
export async function runMaintenance(
|
|
197
|
+
cwd: string,
|
|
198
|
+
deps: BackupRunnerDeps,
|
|
199
|
+
thresholds: MaintenanceThresholds = {},
|
|
200
|
+
): Promise<MaintenanceResult> {
|
|
201
|
+
const repo = resolveAgentGit(cwd)
|
|
202
|
+
if (!repo) return { ok: true, kind: 'no-repo' }
|
|
203
|
+
|
|
204
|
+
const looseThreshold = thresholds.loose ?? MAINTENANCE_LOOSE_THRESHOLD
|
|
205
|
+
const packThreshold = thresholds.packs ?? MAINTENANCE_PACK_THRESHOLD
|
|
206
|
+
const counts = await countObjects(cwd, deps, repo)
|
|
207
|
+
const tasks: string[] = []
|
|
208
|
+
if (counts.loose > looseThreshold) tasks.push('loose-objects')
|
|
209
|
+
if (counts.packs > packThreshold) tasks.push('incremental-repack')
|
|
210
|
+
if (tasks.length === 0) return { ok: true, kind: 'skipped' }
|
|
211
|
+
|
|
212
|
+
const run = await deps.gitSpawn(
|
|
213
|
+
[...repo.gitArgs, ...BOUNDED_PACK_FLAGS, 'maintenance', 'run', ...tasks.map((task) => `--task=${task}`)],
|
|
214
|
+
{ cwd, timeoutMs: MAINTENANCE_TIMEOUT_MS },
|
|
215
|
+
)
|
|
216
|
+
if (run.exitCode !== 0) return { ok: false, kind: 'failed', reason: shortErr(run) }
|
|
217
|
+
return { ok: true, kind: 'ran', tasks }
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Reads the loose-object (`count:`) and pack (`packs:`) counts from
|
|
221
|
+
// `git count-objects -v`. Returns zeroes on any failure so a probe error just
|
|
222
|
+
// skips maintenance rather than aborting it.
|
|
223
|
+
async function countObjects(
|
|
224
|
+
cwd: string,
|
|
225
|
+
deps: BackupRunnerDeps,
|
|
226
|
+
repo: AgentGit,
|
|
227
|
+
): Promise<{ loose: number; packs: number }> {
|
|
228
|
+
const result = await deps.gitSpawn([...repo.gitArgs, 'count-objects', '-v'], { cwd, timeoutMs: COMMIT_TIMEOUT_MS })
|
|
229
|
+
if (result.exitCode !== 0) return { loose: 0, packs: 0 }
|
|
230
|
+
return { loose: readCount(result.stdout, 'count'), packs: readCount(result.stdout, 'packs') }
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function readCount(stdout: string, field: 'count' | 'packs'): number {
|
|
234
|
+
const match = new RegExp(String.raw`^${field}:\s*(\d+)`, 'm').exec(stdout)
|
|
235
|
+
return match?.[1] !== undefined ? Number.parseInt(match[1], 10) : 0
|
|
236
|
+
}
|
|
237
|
+
|
|
155
238
|
// `@{upstream}` resolution failing was previously treated as "no push" — but a
|
|
156
239
|
// fresh agent repo that nobody ran `git push -u` on has a configured `origin`
|
|
157
240
|
// and no tracking ref, so the runner committed forever and never pushed. The
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
|
-
import {
|
|
2
|
+
import { constants } from 'node:fs'
|
|
3
|
+
import { access, chmod, mkdir, mkdtemp, rename, writeFile } from 'node:fs/promises'
|
|
3
4
|
import { dirname, join } from 'node:path'
|
|
4
5
|
|
|
5
6
|
// A GIT_ASKPASS helper git invokes for username/password prompts. The token
|
|
@@ -25,7 +26,7 @@ import { dirname, join } from 'node:path'
|
|
|
25
26
|
// \`github.com.evil/\`, or \`x@github.com.evil/\`. Without the userinfo arm the
|
|
26
27
|
// password prompt falls through to \`exit 1\` and every HTTPS clone/fetch fails
|
|
27
28
|
// with "unable to read askpass response".
|
|
28
|
-
const ASKPASS_SCRIPT = `#!/bin/sh
|
|
29
|
+
export const ASKPASS_SCRIPT = `#!/bin/sh
|
|
29
30
|
case "$1" in
|
|
30
31
|
*//github.com/*|*//github.com\\'*|*//*@github.com/*|*//*@github.com\\'*) : ;;
|
|
31
32
|
*) exit 1 ;;
|
|
@@ -36,40 +37,87 @@ case "$1" in
|
|
|
36
37
|
esac
|
|
37
38
|
`
|
|
38
39
|
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
// a
|
|
42
|
-
//
|
|
43
|
-
|
|
40
|
+
// The sandboxed-bash consumer (the github-cli-auth broker's authenticated git)
|
|
41
|
+
// runs INSIDE the per-tool bwrap sandbox, which `--ro-bind`s /usr but masks /tmp
|
|
42
|
+
// with a tmpfs (src/sandbox/build.ts). So its helper must live under /usr, baked
|
|
43
|
+
// into the image at build time (dockerfile.ts) — the read-only /usr is why
|
|
44
|
+
// ensureGitAskPassHelper early-returns when the file already exists rather than
|
|
45
|
+
// rewriting it. TYPECLAW_GIT_ASKPASS_PATH overrides it for tests/CI.
|
|
46
|
+
export const TYPECLAW_GIT_ASKPASS_PATH = '/usr/local/bin/typeclaw-git-askpass'
|
|
44
47
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
+
// The base is the FIXED real `/tmp`, NOT `os.tmpdir()`: os.tmpdir() honors
|
|
49
|
+
// TMPDIR/TMP/TEMP, so an env pointing it at a model-writable location would let a
|
|
50
|
+
// sandboxed tool replace the cached helper before a later UNSANDBOXED reviewer
|
|
51
|
+
// checkout / backup push executes it with TYPECLAW_GIT_TOKEN in env. A random
|
|
52
|
+
// `mkdtemp` subdir (mode 0700) keeps the name unpredictable.
|
|
53
|
+
const ASKPASS_DIR_BASE = '/tmp/typeclaw-git-askpass-'
|
|
54
|
+
|
|
55
|
+
let defaultDirPromise: Promise<string> | null = null
|
|
56
|
+
|
|
57
|
+
function configuredPath(): string | undefined {
|
|
58
|
+
const path = process.env.TYPECLAW_GIT_ASKPASS_PATH
|
|
59
|
+
return path !== undefined && path !== '' ? path : undefined
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function resolveDefaultDir(): Promise<string> {
|
|
63
|
+
if (defaultDirPromise === null) defaultDirPromise = mkdtemp(ASKPASS_DIR_BASE)
|
|
64
|
+
return defaultDirPromise.catch((err) => {
|
|
65
|
+
defaultDirPromise = null
|
|
66
|
+
throw err
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// The path the SANDBOXED-bash consumer must use: the baked, sandbox-visible /usr
|
|
71
|
+
// helper (or the test/CI override). Unlike the unsandboxed default, this never
|
|
72
|
+
// falls back to /tmp, which the sandbox's tmpfs would hide from model bash.
|
|
73
|
+
export function resolveSandboxGitAskPassPath(): string {
|
|
74
|
+
return configuredPath() ?? TYPECLAW_GIT_ASKPASS_PATH
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// The path UNSANDBOXED execFile consumers (reviewer checkout, backup push) use:
|
|
78
|
+
// a runtime-writable, unpredictable real-/tmp path. They don't run under bwrap, so
|
|
79
|
+
// /tmp masking doesn't apply, and a non-root runtime can write here.
|
|
80
|
+
async function defaultPath(): Promise<string> {
|
|
81
|
+
return configuredPath() ?? join(await resolveDefaultDir(), 'typeclaw-git-askpass')
|
|
48
82
|
}
|
|
49
83
|
|
|
50
|
-
|
|
84
|
+
// Keyed by resolved path: a single shared promise would let the first caller's
|
|
85
|
+
// path win even when a later caller explicitly asks for a different one (the
|
|
86
|
+
// sandbox /usr path and the unsandboxed /tmp path are legitimately distinct).
|
|
87
|
+
const ensurePromises = new Map<string, Promise<string>>()
|
|
51
88
|
|
|
52
89
|
export function resetGitAskPassHelperForTests(): void {
|
|
53
|
-
|
|
90
|
+
ensurePromises.clear()
|
|
91
|
+
defaultDirPromise = null
|
|
54
92
|
}
|
|
55
93
|
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
//
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
94
|
+
// Returns the helper's absolute path, creating it once per path if needed. When
|
|
95
|
+
// an executable file already exists at the resolved path (the baked /usr helper),
|
|
96
|
+
// we return it WITHOUT writing — the read-only /usr would EACCES a rewrite. Only
|
|
97
|
+
// creates when absent: dev, the unsandboxed /tmp path, or a writable test
|
|
98
|
+
// override. Idempotent + race-safe via the per-path promise; the create uses an
|
|
99
|
+
// unpredictable temp opened `wx` (fails on an existing file/symlink) then renamed.
|
|
100
|
+
export async function ensureGitAskPassHelper(path?: string): Promise<string> {
|
|
101
|
+
const resolved = path ?? (await defaultPath())
|
|
102
|
+
const existing = ensurePromises.get(resolved)
|
|
103
|
+
if (existing !== undefined) return existing
|
|
104
|
+
const pending = (async () => {
|
|
105
|
+
try {
|
|
106
|
+
await access(resolved, constants.X_OK)
|
|
107
|
+
return resolved
|
|
108
|
+
} catch {
|
|
109
|
+
// not present (or not executable) — fall through to create it
|
|
110
|
+
}
|
|
111
|
+
await mkdir(dirname(resolved), { recursive: true })
|
|
112
|
+
const tmp = join(dirname(resolved), `.typeclaw-git-askpass.${randomBytes(8).toString('hex')}.tmp`)
|
|
66
113
|
await writeFile(tmp, ASKPASS_SCRIPT, { mode: 0o755, flag: 'wx' })
|
|
67
114
|
await chmod(tmp, 0o755)
|
|
68
|
-
await rename(tmp,
|
|
69
|
-
return
|
|
115
|
+
await rename(tmp, resolved)
|
|
116
|
+
return resolved
|
|
70
117
|
})().catch((err) => {
|
|
71
|
-
|
|
118
|
+
ensurePromises.delete(resolved)
|
|
72
119
|
throw err
|
|
73
120
|
})
|
|
74
|
-
|
|
121
|
+
ensurePromises.set(resolved, pending)
|
|
122
|
+
return pending
|
|
75
123
|
}
|