typeclaw 0.39.3 → 0.39.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/index.ts +10 -14
- package/src/agent/model-fallback.ts +13 -4
- package/src/agent/provider-error.ts +36 -1
- package/src/agent/subagents.ts +29 -2
- package/src/agent/throttle-circuit.ts +50 -0
- package/src/agent/turn-runner.ts +162 -0
- package/src/bundled-plugins/memory/README.md +12 -12
- package/src/bundled-plugins/memory/append-tool.ts +58 -6
- package/src/bundled-plugins/memory/dreaming.ts +1 -1
- package/src/bundled-plugins/memory/fragment-parser.ts +10 -3
- package/src/bundled-plugins/memory/load-memory.ts +28 -2
- package/src/bundled-plugins/memory/memory-logger.ts +14 -2
- package/src/bundled-plugins/memory/search-tool.ts +50 -7
- package/src/bundled-plugins/memory/stream-events.ts +23 -0
- package/src/bundled-plugins/memory/vector/hybrid.ts +9 -2
- package/src/bundled-plugins/security/index.ts +24 -1
- package/src/bundled-plugins/security/policies/cron-promotion.ts +94 -62
- package/src/bundled-plugins/security/policies/plugin-addition.ts +2 -4
- package/src/bundled-plugins/security/policies/role-promotion.ts +2 -4
- package/src/channels/adapters/webex-bot.ts +26 -25
- package/src/channels/adapters/webex-prefetch-limiter.ts +120 -0
- package/src/channels/adapters/webex.ts +27 -30
- package/src/channels/manager.ts +64 -57
- package/src/channels/router.ts +134 -4
- package/src/channels/types.ts +8 -1
- package/src/cli/builtins.ts +6 -32
- package/src/cli/command-meta.ts +93 -0
- package/src/cli/help.ts +24 -0
- package/src/cli/index.ts +34 -10
- package/src/config/config.ts +27 -1
- package/src/container/start.ts +22 -2
- package/src/hostd/daemon.ts +55 -26
- package/src/hostd/spawn.ts +53 -19
- package/src/permissions/permissions.ts +12 -0
- package/src/run/codex-fetch-observer.ts +11 -6
- package/src/secrets/oauth-xai.ts +2 -2
- package/src/server/index.ts +27 -2
- package/src/skills/typeclaw-channels/SKILL.md +1 -1
- package/src/skills/typeclaw-config/SKILL.md +1 -0
- package/src/skills/typeclaw-cron/SKILL.md +9 -4
- package/src/skills/typeclaw-permissions/SKILL.md +18 -6
|
@@ -4,7 +4,16 @@ import path from 'node:path'
|
|
|
4
4
|
import { parseCronJson, type ParsedCronJob } from '@/cron'
|
|
5
5
|
|
|
6
6
|
import type { SecuritySeverity } from '../permissions'
|
|
7
|
-
import {
|
|
7
|
+
import type { SecurityBlock } from '../policy'
|
|
8
|
+
|
|
9
|
+
// True iff the caller may legitimately schedule deferred work that fires as
|
|
10
|
+
// `targetRole` — i.e. `targetRole` grants no permission the caller's own role
|
|
11
|
+
// lacks (capability dominance / permission-subset). The caller supplies this;
|
|
12
|
+
// the production wiring in index.ts compares resolved permission sets, NOT the
|
|
13
|
+
// coarse severity tower (which ranks every custom role equal and would let one
|
|
14
|
+
// custom role launder into another). Fails closed: an unknown caller or target
|
|
15
|
+
// role returns false, so the change is treated as an escalation and blocked.
|
|
16
|
+
export type CanScheduleAs = (targetRole: string | undefined) => boolean
|
|
8
17
|
|
|
9
18
|
export const GUARD_CRON_PROMOTION = 'cronPromotion'
|
|
10
19
|
// Classified `medium` (silent-attack axis). Originally `high`; reclassified
|
|
@@ -18,12 +27,12 @@ export const GUARD_CRON_PROMOTION = 'cronPromotion'
|
|
|
18
27
|
// audience-leak.
|
|
19
28
|
//
|
|
20
29
|
// Net effect on the role-tower model: owner and trusted both bypass
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
30
|
+
// (they carry `security.bypass.medium`); member and guest are gated by
|
|
31
|
+
// the caller-role-aware predicate below. This guard does NOT honor
|
|
32
|
+
// `acknowledgeGuards` — an ack flag from an under-privileged session is
|
|
33
|
+
// ignored. Bypassing is a property of the caller's resolved role, not a
|
|
34
|
+
// tool-arg flag, because an args-level ack would let any member/guest
|
|
35
|
+
// launder itself past the guard (it inspected args, not permissions).
|
|
27
36
|
//
|
|
28
37
|
// Cron is the deferred-execution sibling of `roles`. Every cron job
|
|
29
38
|
// carries a `scheduledByRole` field that the runtime stamps into the
|
|
@@ -32,57 +41,64 @@ export const GUARD_CRON_PROMOTION = 'cronPromotion'
|
|
|
32
41
|
// table"). The `parseCronFile` boot gate rejects entries without
|
|
33
42
|
// `scheduledByRole`, but it accepts any role name the file declares.
|
|
34
43
|
//
|
|
35
|
-
// Concrete breach pattern blocked
|
|
36
|
-
//
|
|
44
|
+
// Concrete breach pattern blocked: a `member`-role agent that can
|
|
45
|
+
// `write` `cron.json` authors a brand-new job with
|
|
37
46
|
// `"scheduledByRole": "owner"` and a prompt that does whatever the
|
|
38
47
|
// agent's tool surface allows when running as owner. The cron consumer
|
|
39
|
-
// fires it on schedule; the firing session resolves to `owner
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
// — member does not carry `bypass.medium`.
|
|
48
|
+
// fires it on schedule; the firing session resolves to `owner`. The
|
|
49
|
+
// agent has laundered itself into owner via the schedule. This guard
|
|
50
|
+
// blocks it because `canScheduleAs('owner')` is false for a member.
|
|
43
51
|
//
|
|
44
52
|
// Same two-step shape as `gitRemoteTainted`: "do a privileged write
|
|
45
53
|
// now, run the privileged thing later." This guard blocks the first
|
|
46
|
-
// step.
|
|
54
|
+
// step. It is the deferred-write analogue of the `spawnedByRole` cap in
|
|
55
|
+
// src/agent/tools/subagent-access.ts: a caller may only commit deferred
|
|
56
|
+
// work that fires at a role it could already act as.
|
|
47
57
|
//
|
|
48
|
-
// What counts as a cron promotion (
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
//
|
|
58
|
+
// What counts as a cron promotion (a finding is raised only when the
|
|
59
|
+
// change introduces deferred authority the caller lacks — i.e. the
|
|
60
|
+
// relevant `scheduledByRole` is NOT schedulable by the caller, where
|
|
61
|
+
// `canScheduleAs(R)` means R's permission set is a SUBSET of the
|
|
62
|
+
// caller's (capability dominance), fail-closed on unknown roles):
|
|
63
|
+
// 1. A new job (by id) was added whose `scheduledByRole` the caller
|
|
64
|
+
// cannot schedule as. Adding a job at-or-below the caller's own
|
|
65
|
+
// role grants nothing new and is allowed.
|
|
66
|
+
// 2. An existing job's `scheduledByRole` was raised to a value the
|
|
67
|
+
// caller cannot schedule as. Lowering it, or a lateral move within
|
|
68
|
+
// the caller's reach, is allowed.
|
|
53
69
|
// 3. An existing job's EXECUTABLE BODY changed — `kind`, `prompt`,
|
|
54
|
-
// `command`, `subagent`, or `payload
|
|
55
|
-
//
|
|
56
|
-
// is the same deferred-laundering
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
// consumer
|
|
63
|
-
//
|
|
64
|
-
// separately or are audit metadata
|
|
65
|
-
// 4. An existing job had `enabled: false` flipped to true
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
// same sense as adding the job fresh.
|
|
70
|
+
// `command`, `subagent`, or `payload` — AND the job fires above
|
|
71
|
+
// the caller either before or after the change. Rewriting the body
|
|
72
|
+
// of an already-privileged job is the same deferred-laundering
|
|
73
|
+
// attack as job creation (Oracle PR #305 critical finding): the
|
|
74
|
+
// cron consumer fires the new body as the stamped role. Requiring
|
|
75
|
+
// BOTH old and new roles to be within the caller's reach preserves
|
|
76
|
+
// that defense — a member cannot rewrite an owner-stamped body even
|
|
77
|
+
// though `scheduledByRole` is untouched. The fields chosen are
|
|
78
|
+
// exactly those the cron consumer uses to decide what executes;
|
|
79
|
+
// provenance/identity fields (id, scheduledByRole,
|
|
80
|
+
// scheduledByOrigin) are handled separately or are audit metadata.
|
|
81
|
+
// 4. An existing job had `enabled: false` flipped to true while it
|
|
82
|
+
// fires above the caller (same both-sides reach check as body
|
|
83
|
+
// changes). Re-enabling a job at-or-below the caller is allowed.
|
|
69
84
|
//
|
|
70
|
-
// What does NOT count (allowed
|
|
85
|
+
// What does NOT count (allowed):
|
|
86
|
+
// - Any change (add / role-change / body / re-enable) whose resulting
|
|
87
|
+
// job fires at or below the caller's own role: it grants the caller
|
|
88
|
+
// no authority it doesn't already have.
|
|
71
89
|
// - Removing a job entirely.
|
|
72
90
|
// - Changing `schedule` or `timezone` on an existing job (cadence
|
|
73
91
|
// decisions; do not change what runs, only when).
|
|
74
|
-
// - Setting `enabled: true -> false` (disabling a
|
|
75
|
-
//
|
|
76
|
-
//
|
|
77
|
-
//
|
|
78
|
-
// this branch is unreachable in practice; the guard treats it as
|
|
79
|
-
// a non-finding for forward compatibility).
|
|
92
|
+
// - Setting `enabled: true -> false` (disabling is a REDUCTION).
|
|
93
|
+
// - A job with no `scheduledByRole` (`undefined`) is treated as NOT
|
|
94
|
+
// schedulable (fail-closed), so the schema-rejected unset case stays
|
|
95
|
+
// blocked rather than slipping through as a non-finding.
|
|
80
96
|
//
|
|
81
|
-
// Failure-
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
//
|
|
97
|
+
// Failure-closed on caller role: an unknown/incomparable caller role
|
|
98
|
+
// makes `canScheduleAs` return false, so every change is treated as an
|
|
99
|
+
// escalation and blocked. Failure-open on file read is unchanged: if the
|
|
100
|
+
// existing `cron.json` cannot be read or parsed, every proposed job is
|
|
101
|
+
// treated as new and then subjected to the same caller-role check.
|
|
86
102
|
export const GUARD_CRON_PROMOTION_SEVERITY: SecuritySeverity = 'medium'
|
|
87
103
|
|
|
88
104
|
export type CronPromotionFinding =
|
|
@@ -95,8 +111,9 @@ export async function checkCronPromotionGuard(options: {
|
|
|
95
111
|
tool: string
|
|
96
112
|
args: Record<string, unknown>
|
|
97
113
|
agentDir: string
|
|
114
|
+
canScheduleAs: CanScheduleAs
|
|
98
115
|
}): Promise<SecurityBlock | undefined> {
|
|
99
|
-
const { tool, args, agentDir } = options
|
|
116
|
+
const { tool, args, agentDir, canScheduleAs } = options
|
|
100
117
|
if (tool !== 'write' && tool !== 'edit') return undefined
|
|
101
118
|
|
|
102
119
|
const rawPath = args.path
|
|
@@ -106,8 +123,6 @@ export async function checkCronPromotionGuard(options: {
|
|
|
106
123
|
const isCronJson = await pathIsCronJson(agentDir, targetPath)
|
|
107
124
|
if (!isCronJson) return undefined
|
|
108
125
|
|
|
109
|
-
if (isGuardAcknowledged(args, GUARD_CRON_PROMOTION)) return undefined
|
|
110
|
-
|
|
111
126
|
const editRefusal = refuseRiskyEdit(tool, args, targetPath)
|
|
112
127
|
if (editRefusal) return editRefusal
|
|
113
128
|
|
|
@@ -118,7 +133,7 @@ export async function checkCronPromotionGuard(options: {
|
|
|
118
133
|
if (newJobs === undefined) return undefined
|
|
119
134
|
|
|
120
135
|
const oldJobs = await readExistingJobs(targetPath)
|
|
121
|
-
const findings = diffJobs(oldJobs, newJobs)
|
|
136
|
+
const findings = diffJobs(oldJobs, newJobs, canScheduleAs)
|
|
122
137
|
if (findings.length === 0) return undefined
|
|
123
138
|
|
|
124
139
|
return {
|
|
@@ -208,7 +223,21 @@ async function readExistingJobs(targetPath: string): Promise<readonly ParsedCron
|
|
|
208
223
|
return result.file.jobs
|
|
209
224
|
}
|
|
210
225
|
|
|
211
|
-
|
|
226
|
+
// A finding is a *promotion* only when the change introduces deferred authority
|
|
227
|
+
// the caller could not already wield. `canScheduleAs(R)` is true when R's
|
|
228
|
+
// permission set is a subset of the caller's, so a change that lands (and, for
|
|
229
|
+
// mutations, started) at a role granting nothing the caller lacks is not
|
|
230
|
+
// flagged. The laundering shapes still trip every branch: a member
|
|
231
|
+
// adding/role-changing a job to `owner` fails `canScheduleAs('owner')`,
|
|
232
|
+
// rewriting/re-enabling an already-`owner`-stamped body fails the prior-role
|
|
233
|
+
// check, and a custom role scheduling as a different custom role that carries
|
|
234
|
+
// an extra permission fails the subset check — none are schedulable, so all
|
|
235
|
+
// stay blocked.
|
|
236
|
+
export function diffJobs(
|
|
237
|
+
before: readonly ParsedCronJob[],
|
|
238
|
+
after: readonly ParsedCronJob[],
|
|
239
|
+
canScheduleAs: CanScheduleAs,
|
|
240
|
+
): CronPromotionFinding[] {
|
|
212
241
|
const findings: CronPromotionFinding[] = []
|
|
213
242
|
const beforeById = new Map<string, ParsedCronJob>()
|
|
214
243
|
for (const job of before) beforeById.set(job.id, job)
|
|
@@ -217,15 +246,17 @@ export function diffJobs(before: readonly ParsedCronJob[], after: readonly Parse
|
|
|
217
246
|
const prior = beforeById.get(job.id)
|
|
218
247
|
const newRole = job.scheduledByRole
|
|
219
248
|
if (prior === undefined) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
249
|
+
if (!canScheduleAs(newRole)) {
|
|
250
|
+
findings.push({
|
|
251
|
+
kind: 'job-added',
|
|
252
|
+
id: job.id,
|
|
253
|
+
scheduledByRole: newRole ?? '<unset>',
|
|
254
|
+
})
|
|
255
|
+
}
|
|
225
256
|
continue
|
|
226
257
|
}
|
|
227
258
|
const oldRole = prior.scheduledByRole
|
|
228
|
-
if (oldRole !== newRole) {
|
|
259
|
+
if (oldRole !== newRole && !canScheduleAs(newRole)) {
|
|
229
260
|
findings.push({
|
|
230
261
|
kind: 'role-changed',
|
|
231
262
|
id: job.id,
|
|
@@ -233,8 +264,9 @@ export function diffJobs(before: readonly ParsedCronJob[], after: readonly Parse
|
|
|
233
264
|
to: newRole ?? '<unset>',
|
|
234
265
|
})
|
|
235
266
|
}
|
|
267
|
+
const withinReach = canScheduleAs(oldRole) && canScheduleAs(newRole)
|
|
236
268
|
const bodyDelta = diffJobBody(prior, job)
|
|
237
|
-
if (bodyDelta.length > 0) {
|
|
269
|
+
if (bodyDelta.length > 0 && !withinReach) {
|
|
238
270
|
findings.push({
|
|
239
271
|
kind: 'body-changed',
|
|
240
272
|
id: job.id,
|
|
@@ -242,7 +274,7 @@ export function diffJobs(before: readonly ParsedCronJob[], after: readonly Parse
|
|
|
242
274
|
fields: bodyDelta,
|
|
243
275
|
})
|
|
244
276
|
}
|
|
245
|
-
if (isPreviouslyDisabled(prior) && !isPreviouslyDisabled(job)) {
|
|
277
|
+
if (isPreviouslyDisabled(prior) && !isPreviouslyDisabled(job) && !withinReach) {
|
|
246
278
|
findings.push({
|
|
247
279
|
kind: 'enabled-flipped',
|
|
248
280
|
id: job.id,
|
|
@@ -329,9 +361,9 @@ function buildBlockReason(tool: string, targetPath: string, findings: readonly C
|
|
|
329
361
|
}
|
|
330
362
|
}
|
|
331
363
|
return [
|
|
332
|
-
`Guard \`${GUARD_CRON_PROMOTION}\` blocked ${tool} on ${sanitizeForReason(targetPath)}: this change
|
|
333
|
-
'Cron jobs carry `scheduledByRole`, which the runtime stamps into the firing session\'s origin. Adding a job
|
|
334
|
-
|
|
364
|
+
`Guard \`${GUARD_CRON_PROMOTION}\` blocked ${tool} on ${sanitizeForReason(targetPath)}: this change schedules deferred work above your role — ${lines.join('; ')}.`,
|
|
365
|
+
'Cron jobs carry `scheduledByRole`, which the runtime stamps into the firing session\'s origin. Adding a job, raising its scheduledByRole, or rewriting/re-enabling a job that already fires above your role is the deferred form of the `rolePromotion` attack: "schedule a privileged prompt now, the cron consumer runs it as that role later." A change that fires at or below your own role grants nothing new and is allowed; this block means the resulting (or prior) role outranks yours.',
|
|
366
|
+
'This cannot be acknowledged away from an under-privileged session. The operator must make the change from a session that already resolves to a role at least as high as the cron job — TUI (owner), or a role granted `security.bypass.medium` — or claim the role out-of-band via `typeclaw role claim`.',
|
|
335
367
|
].join(' ')
|
|
336
368
|
}
|
|
337
369
|
|
|
@@ -5,7 +5,7 @@ import { parseConfigJson } from '@/config'
|
|
|
5
5
|
import { splitPluginEntrySpec } from '@/plugin'
|
|
6
6
|
|
|
7
7
|
import type { SecuritySeverity } from '../permissions'
|
|
8
|
-
import {
|
|
8
|
+
import type { SecurityBlock } from '../policy'
|
|
9
9
|
|
|
10
10
|
export const GUARD_PLUGIN_ADDITION = 'pluginAddition'
|
|
11
11
|
// Classified `medium` (silent-attack axis), same tier and rationale as
|
|
@@ -57,8 +57,6 @@ export async function checkPluginAdditionGuard(options: {
|
|
|
57
57
|
const targetPath = path.resolve(agentDir, rawPath)
|
|
58
58
|
if (!(await pathIsTypeclawJson(agentDir, targetPath))) return undefined
|
|
59
59
|
|
|
60
|
-
if (isGuardAcknowledged(args, GUARD_PLUGIN_ADDITION)) return undefined
|
|
61
|
-
|
|
62
60
|
const editRefusal = refuseRiskyEdit(tool, args, targetPath)
|
|
63
61
|
if (editRefusal) return editRefusal
|
|
64
62
|
|
|
@@ -205,7 +203,7 @@ function buildBlockReason(tool: string, targetPath: string, findings: readonly P
|
|
|
205
203
|
return [
|
|
206
204
|
`Guard \`${GUARD_PLUGIN_ADDITION}\` blocked ${tool} on ${sanitizeForReason(targetPath)}: this change introduces a deferred host-code-execution grant — ${lines.join('; ')}.`,
|
|
207
205
|
"Plugin entries in typeclaw.json#plugins are materialized into package.json and installed by the next host `typeclaw start`, where npm lifecycle scripts run as the operator on the host. Even an `owner` operating from TUI must not silently add plugins on behalf of a channel message: the canonical attack is a prompt-injected agent writing a malicious package name into plugins[] so the operator's next start runs its postinstall.",
|
|
208
|
-
|
|
206
|
+
'This cannot be acknowledged away from an under-privileged session. The operator must make the change from a session that already resolves to `owner`/`trusted` (TUI, or a role granted `security.bypass.medium`), or claim the role out-of-band via `typeclaw role claim`.',
|
|
209
207
|
].join(' ')
|
|
210
208
|
}
|
|
211
209
|
|
|
@@ -5,7 +5,7 @@ import { parseConfigJson } from '@/config'
|
|
|
5
5
|
import { isBuiltinRoleName, type RoleConfig, type RolesConfig } from '@/permissions'
|
|
6
6
|
|
|
7
7
|
import type { SecuritySeverity } from '../permissions'
|
|
8
|
-
import {
|
|
8
|
+
import type { SecurityBlock } from '../policy'
|
|
9
9
|
|
|
10
10
|
export const GUARD_ROLE_PROMOTION = 'rolePromotion'
|
|
11
11
|
// Classified `medium` (silent-attack axis). Originally `high`; reclassified
|
|
@@ -88,8 +88,6 @@ export async function checkRolePromotionGuard(options: {
|
|
|
88
88
|
const isTypeclawJson = await pathIsTypeclawJson(agentDir, targetPath)
|
|
89
89
|
if (!isTypeclawJson) return undefined
|
|
90
90
|
|
|
91
|
-
if (isGuardAcknowledged(args, GUARD_ROLE_PROMOTION)) return undefined
|
|
92
|
-
|
|
93
91
|
const editRefusal = refuseRiskyEdit(tool, args, targetPath)
|
|
94
92
|
if (editRefusal) return editRefusal
|
|
95
93
|
|
|
@@ -378,7 +376,7 @@ function buildBlockReason(tool: string, targetPath: string, findings: readonly R
|
|
|
378
376
|
return [
|
|
379
377
|
`Guard \`${GUARD_ROLE_PROMOTION}\` blocked ${tool} on ${sanitizeForReason(targetPath)}: this change is a privilege escalation — ${lines.join('; ')}.`,
|
|
380
378
|
'Granting `owner` / `trusted` (or widening any role) gives the matched actor security-bypass capabilities, cron scheduling, channel respond, and operator-only subagent spawn. Even an operator running from TUI must not silently rewrite the access-control table based on a channel message: the canonical attack is a member-role speaker socially-engineering the agent into adding their own author-id to `roles.owner.match[]`, which the schema check accepts as valid.',
|
|
381
|
-
|
|
379
|
+
'This cannot be acknowledged away from an under-privileged session. The operator must make the change from a session that already resolves to `owner`/`trusted` (TUI, or a role granted `security.bypass.medium`), or claim the role out-of-band via `typeclaw role claim`.',
|
|
382
380
|
].join(' ')
|
|
383
381
|
}
|
|
384
382
|
|
|
@@ -33,6 +33,7 @@ import { createWebexChannelNameResolver } from './webex-bot-channel-resolver'
|
|
|
33
33
|
import { classifyInbound, type InboundDropReason, type WebexInboundMessage } from './webex-bot-classify'
|
|
34
34
|
import { enrichWebexMessageReference } from './webex-bot-reference'
|
|
35
35
|
import { resolveWebexBodyText } from './webex-format'
|
|
36
|
+
import { createWebexPrefetchLimiter, isWebexRateLimitError, type WebexPrefetchLimiter } from './webex-prefetch-limiter'
|
|
36
37
|
|
|
37
38
|
export type WebexBotAdapterLogger = {
|
|
38
39
|
info: (msg: string) => void
|
|
@@ -139,27 +140,39 @@ async function defaultReadFile(path: string): Promise<WebexOutboundFile> {
|
|
|
139
140
|
return { content: Bun.file(path), filename: path.split('/').pop() ?? 'attachment' }
|
|
140
141
|
}
|
|
141
142
|
|
|
142
|
-
// See webex.ts: the bot client has no AbortSignal either, so a hung listMessages
|
|
143
|
-
// is bounded by a tighter internal deadline than the router's 5s history ceiling.
|
|
144
|
-
const WEBEX_HISTORY_COLD_FETCH_TIMEOUT_MS = 2500
|
|
145
|
-
|
|
146
143
|
export function createWebexHistoryCallback(deps: {
|
|
147
144
|
client: Pick<WebexBotClient, 'listMessages'>
|
|
148
145
|
logger: WebexBotAdapterLogger
|
|
149
146
|
botPersonIdRef: () => string | null
|
|
150
|
-
|
|
147
|
+
limiter?: WebexPrefetchLimiter
|
|
151
148
|
}): HistoryCallback {
|
|
152
|
-
const
|
|
149
|
+
const limiter = deps.limiter ?? createWebexPrefetchLimiter()
|
|
150
|
+
const listMessages = (args: FetchHistoryArgs) =>
|
|
151
|
+
deps.client.listMessages(args.chat, { max: clampLimit(args.limit, 100) })
|
|
152
|
+
const toHistory = (messages: WebexMessage[]): FetchHistoryResult => ({
|
|
153
|
+
ok: true,
|
|
154
|
+
messages: messages.map((m) => mapWebexHistoryMessage(m, deps.botPersonIdRef())).reverse(),
|
|
155
|
+
})
|
|
153
156
|
return async (args: FetchHistoryArgs): Promise<FetchHistoryResult> => {
|
|
154
157
|
try {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
158
|
+
// Only best-effort reads (cold-start prefetch, membership fallback) submit
|
|
159
|
+
// to the per-room limiter. Explicit channel_history reads bypass it so a
|
|
160
|
+
// user-initiated look-back is never declined by prefetch backpressure.
|
|
161
|
+
if (args.prefetch !== true) {
|
|
162
|
+
return toHistory(await listMessages(args))
|
|
163
|
+
}
|
|
164
|
+
const outcome = await limiter.run(args.chat, () => listMessages(args))
|
|
165
|
+
if (!outcome.admitted) {
|
|
166
|
+
deps.logger.info('[webex-bot] history prefetch skipped: rate-limit backpressure')
|
|
167
|
+
return { ok: false, error: 'prefetch skipped: rate-limit backpressure', skipReason: 'rate-limited' }
|
|
168
|
+
}
|
|
169
|
+
return toHistory(outcome.value)
|
|
161
170
|
} catch (err) {
|
|
162
171
|
const message = describe(err)
|
|
172
|
+
if (args.prefetch === true && isWebexRateLimitError(err)) {
|
|
173
|
+
deps.logger.info(`[webex-bot] history prefetch skipped: rate limited (${message})`)
|
|
174
|
+
return { ok: false, error: message, skipReason: 'rate-limited' }
|
|
175
|
+
}
|
|
163
176
|
deps.logger.warn(`[webex-bot] history fetch failed: ${message}`)
|
|
164
177
|
return { ok: false, error: message }
|
|
165
178
|
}
|
|
@@ -204,7 +217,7 @@ export function createWebexMembershipResolver(deps: {
|
|
|
204
217
|
} catch (err) {
|
|
205
218
|
deps.logger.warn(`[webex-bot] membership room=${key.chat} failed: ${describe(err)}; deriving from recent history`)
|
|
206
219
|
return await deriveMembershipFromHistory({
|
|
207
|
-
fetchHistory: (limit) => deps.historyCallback({ chat: key.chat, thread: key.thread, limit }),
|
|
220
|
+
fetchHistory: (limit) => deps.historyCallback({ chat: key.chat, thread: key.thread, limit, prefetch: true }),
|
|
208
221
|
now,
|
|
209
222
|
})
|
|
210
223
|
}
|
|
@@ -465,18 +478,6 @@ function describe(err: unknown): string {
|
|
|
465
478
|
return err instanceof Error ? err.message : String(err)
|
|
466
479
|
}
|
|
467
480
|
|
|
468
|
-
async function raceWithTimeout<T>(work: Promise<T>, ms: number, label: string): Promise<T> {
|
|
469
|
-
let timer: ReturnType<typeof setTimeout> | null = null
|
|
470
|
-
const timeout = new Promise<never>((_, reject) => {
|
|
471
|
-
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms)
|
|
472
|
-
})
|
|
473
|
-
try {
|
|
474
|
-
return await Promise.race([work, timeout])
|
|
475
|
-
} finally {
|
|
476
|
-
if (timer !== null) clearTimeout(timer)
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
|
|
480
481
|
function dropHint(reason: InboundDropReason): string {
|
|
481
482
|
switch (reason) {
|
|
482
483
|
case 'empty_content':
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// Webex's internal conversation API rate-limits reads PER ROOM, not per account:
|
|
2
|
+
// hammering one room's history returns HTTP 429 (x-ratelimit-limit ~3, Retry-After
|
|
3
|
+
// up to ~25s) while a different room served by the same token stays unaffected
|
|
4
|
+
// (verified empirically — saturating room A produced zero 429s on room B). The
|
|
5
|
+
// upstream agent-messenger SDK retries 429s, but its bucket tracker is reactive
|
|
6
|
+
// (it learns the limit only after a 429 lands) and its 3-retry budget cannot
|
|
7
|
+
// outlast a 25s window, so same-room pile-up — cold-start prefetch plus the
|
|
8
|
+
// membership resolver's deriveMembershipFromHistory fallback, both reading the
|
|
9
|
+
// same room — overflows the bucket and fails the prefetch.
|
|
10
|
+
//
|
|
11
|
+
// History prefetch is a context improvement, not liveness — the router skips it
|
|
12
|
+
// and the agent can call channel_history on demand later. So the fix is keyed by
|
|
13
|
+
// room: bound concurrent reads PER ROOM (independent rooms never wait on each
|
|
14
|
+
// other) and keep cold-start non-blocking with bounded admission ("admit fast or
|
|
15
|
+
// skip"), because starting then abandoning a listMessages we cannot cancel (the
|
|
16
|
+
// SDK exposes no AbortSignal) would only burn more quota on an orphaned fetch.
|
|
17
|
+
|
|
18
|
+
export type WebexPrefetchLimiter = {
|
|
19
|
+
// Runs `work` under the permit pool for `key` (the room id). If no slot frees
|
|
20
|
+
// within `admitTimeoutMs`, resolves to `{ admitted: false }` WITHOUT starting
|
|
21
|
+
// `work` — the caller then skips the prefetch. On admission, runs `work` and
|
|
22
|
+
// returns its result. Distinct keys never contend.
|
|
23
|
+
run<T>(key: string, work: () => Promise<T>): Promise<{ admitted: true; value: T } | { admitted: false }>
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type WebexPrefetchLimiterOptions = {
|
|
27
|
+
// Max concurrent reads PER ROOM. Defaults to 2: Webex's per-room cap is ~3, so
|
|
28
|
+
// 2 leaves headroom for the SDK's own retry traffic while still letting a
|
|
29
|
+
// prefetch and a membership-fallback read of the same room overlap.
|
|
30
|
+
concurrency?: number
|
|
31
|
+
// How long a queued caller waits for a slot before giving up and skipping.
|
|
32
|
+
// Defaults to 2500ms, matching the prior cold-fetch fail-fast posture so a
|
|
33
|
+
// backlog never stalls cold-start past the router's history ceiling.
|
|
34
|
+
admitTimeoutMs?: number
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const DEFAULT_CONCURRENCY = 2
|
|
38
|
+
const DEFAULT_ADMIT_TIMEOUT_MS = 2500
|
|
39
|
+
|
|
40
|
+
export function createWebexPrefetchLimiter(options: WebexPrefetchLimiterOptions = {}): WebexPrefetchLimiter {
|
|
41
|
+
const concurrency = Math.max(1, Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY))
|
|
42
|
+
const admitTimeoutMs = options.admitTimeoutMs ?? DEFAULT_ADMIT_TIMEOUT_MS
|
|
43
|
+
|
|
44
|
+
type Pool = { active: number; waiters: Array<() => void> }
|
|
45
|
+
const pools = new Map<string, Pool>()
|
|
46
|
+
|
|
47
|
+
const poolFor = (key: string): Pool => {
|
|
48
|
+
let pool = pools.get(key)
|
|
49
|
+
if (pool === undefined) {
|
|
50
|
+
pool = { active: 0, waiters: [] }
|
|
51
|
+
pools.set(key, pool)
|
|
52
|
+
}
|
|
53
|
+
return pool
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const release = (key: string): void => {
|
|
57
|
+
const pool = pools.get(key)
|
|
58
|
+
if (pool === undefined) return
|
|
59
|
+
pool.active--
|
|
60
|
+
const next = pool.waiters.shift()
|
|
61
|
+
if (next) next()
|
|
62
|
+
// Drop idle pools so a long-lived limiter does not accumulate one entry per
|
|
63
|
+
// room ever seen.
|
|
64
|
+
else if (pool.active === 0 && pool.waiters.length === 0) pools.delete(key)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const acquire = (key: string): Promise<boolean> => {
|
|
68
|
+
const pool = poolFor(key)
|
|
69
|
+
if (pool.active < concurrency) {
|
|
70
|
+
pool.active++
|
|
71
|
+
return Promise.resolve(true)
|
|
72
|
+
}
|
|
73
|
+
return new Promise<boolean>((resolve) => {
|
|
74
|
+
let settled = false
|
|
75
|
+
const timer = setTimeout(() => {
|
|
76
|
+
if (settled) return
|
|
77
|
+
settled = true
|
|
78
|
+
const idx = pool.waiters.indexOf(grant)
|
|
79
|
+
if (idx >= 0) pool.waiters.splice(idx, 1)
|
|
80
|
+
resolve(false)
|
|
81
|
+
}, admitTimeoutMs)
|
|
82
|
+
const grant = (): void => {
|
|
83
|
+
if (settled) {
|
|
84
|
+
// Admitted after we already gave up: hand the freed slot to the next
|
|
85
|
+
// waiter so a timed-out grant never leaks a permanently-held slot.
|
|
86
|
+
release(key)
|
|
87
|
+
return
|
|
88
|
+
}
|
|
89
|
+
settled = true
|
|
90
|
+
clearTimeout(timer)
|
|
91
|
+
pool.active++
|
|
92
|
+
resolve(true)
|
|
93
|
+
}
|
|
94
|
+
pool.waiters.push(grant)
|
|
95
|
+
})
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
async run<T>(key: string, work: () => Promise<T>): Promise<{ admitted: true; value: T } | { admitted: false }> {
|
|
100
|
+
const admitted = await acquire(key)
|
|
101
|
+
if (!admitted) return { admitted: false }
|
|
102
|
+
try {
|
|
103
|
+
return { admitted: true, value: await work() }
|
|
104
|
+
} finally {
|
|
105
|
+
release(key)
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// The SDK throws WebexError with code 'rate_limited' once 429 retries are
|
|
112
|
+
// exhausted, and 'http_429' for a raw 429 that bypassed the retry path. Both mean
|
|
113
|
+
// the same thing to us: an expected, transient, non-fatal rate-limit skip — not an
|
|
114
|
+
// auth/network failure worth a warn. Matching on the SDK's code field (rather than
|
|
115
|
+
// the human message) keeps this stable across SDK message-wording changes.
|
|
116
|
+
export function isWebexRateLimitError(err: unknown): boolean {
|
|
117
|
+
if (typeof err !== 'object' || err === null) return false
|
|
118
|
+
const code = (err as { code?: unknown }).code
|
|
119
|
+
return code === 'rate_limited' || code === 'http_429'
|
|
120
|
+
}
|
|
@@ -37,6 +37,7 @@ import type { WebexAccountRecord } from '@/secrets/schema'
|
|
|
37
37
|
import { createWebexChannelNameResolver } from './webex-channel-resolver'
|
|
38
38
|
import { classifyInbound, type InboundDropReason, type WebexInboundMessage } from './webex-classify'
|
|
39
39
|
import { resolveWebexBodyText } from './webex-format'
|
|
40
|
+
import { createWebexPrefetchLimiter, isWebexRateLimitError, type WebexPrefetchLimiter } from './webex-prefetch-limiter'
|
|
40
41
|
import { enrichWebexMessageReference } from './webex-reference'
|
|
41
42
|
|
|
42
43
|
export type WebexAdapterLogger = {
|
|
@@ -187,32 +188,40 @@ export function createTypingCallback(deps: {
|
|
|
187
188
|
}
|
|
188
189
|
}
|
|
189
190
|
|
|
190
|
-
// The Webex REST/internal client offers no AbortSignal, so a hung `listMessages`
|
|
191
|
-
// would otherwise block cold-start prefetch for the router's full 5s history
|
|
192
|
-
// ceiling before degrading. Racing it against a tighter internal deadline lets
|
|
193
|
-
// prefetch fall back to membership-derivation (or skip) seconds sooner, mirroring
|
|
194
|
-
// the cold-fetch fail-fast posture the membership resolver already uses.
|
|
195
|
-
const WEBEX_HISTORY_COLD_FETCH_TIMEOUT_MS = 2500
|
|
196
|
-
|
|
197
191
|
export function createWebexHistoryCallback(deps: {
|
|
198
192
|
client: Pick<WebexClient, 'listMessages'>
|
|
199
193
|
logger: WebexAdapterLogger
|
|
200
194
|
botPersonIdRef: () => string | null
|
|
201
|
-
|
|
195
|
+
limiter?: WebexPrefetchLimiter
|
|
202
196
|
}): HistoryCallback {
|
|
203
|
-
const
|
|
197
|
+
const limiter = deps.limiter ?? createWebexPrefetchLimiter()
|
|
198
|
+
const listMessages = (args: FetchHistoryArgs) =>
|
|
199
|
+
deps.client.listMessages(args.chat, { max: clampLimit(args.limit, 100) })
|
|
200
|
+
const toHistory = (messages: WebexMessage[]): FetchHistoryResult => {
|
|
201
|
+
const authorById = new Map(messages.map((m) => [m.ref, m.personRef]))
|
|
202
|
+
const botPersonId = deps.botPersonIdRef()
|
|
203
|
+
return { ok: true, messages: messages.map((m) => mapWebexHistoryMessage(m, botPersonId, authorById)).reverse() }
|
|
204
|
+
}
|
|
204
205
|
return async (args: FetchHistoryArgs): Promise<FetchHistoryResult> => {
|
|
205
206
|
try {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
const
|
|
213
|
-
|
|
207
|
+
// Only best-effort reads (cold-start prefetch, membership fallback) submit
|
|
208
|
+
// to the per-room limiter. Explicit channel_history reads bypass it so a
|
|
209
|
+
// user-initiated look-back is never declined by prefetch backpressure.
|
|
210
|
+
if (args.prefetch !== true) {
|
|
211
|
+
return toHistory(await listMessages(args))
|
|
212
|
+
}
|
|
213
|
+
const outcome = await limiter.run(args.chat, () => listMessages(args))
|
|
214
|
+
if (!outcome.admitted) {
|
|
215
|
+
deps.logger.info('[webex] history prefetch skipped: rate-limit backpressure')
|
|
216
|
+
return { ok: false, error: 'prefetch skipped: rate-limit backpressure', skipReason: 'rate-limited' }
|
|
217
|
+
}
|
|
218
|
+
return toHistory(outcome.value)
|
|
214
219
|
} catch (err) {
|
|
215
220
|
const message = describe(err)
|
|
221
|
+
if (args.prefetch === true && isWebexRateLimitError(err)) {
|
|
222
|
+
deps.logger.info(`[webex] history prefetch skipped: rate limited (${message})`)
|
|
223
|
+
return { ok: false, error: message, skipReason: 'rate-limited' }
|
|
224
|
+
}
|
|
216
225
|
deps.logger.warn(`[webex] history fetch failed: ${message}`)
|
|
217
226
|
return { ok: false, error: message }
|
|
218
227
|
}
|
|
@@ -257,7 +266,7 @@ export function createWebexMembershipResolver(deps: {
|
|
|
257
266
|
} catch (err) {
|
|
258
267
|
deps.logger.warn(`[webex] membership room=${key.chat} failed: ${describe(err)}; deriving from recent history`)
|
|
259
268
|
return await deriveMembershipFromHistory({
|
|
260
|
-
fetchHistory: (limit) => deps.historyCallback({ chat: key.chat, thread: key.thread, limit }),
|
|
269
|
+
fetchHistory: (limit) => deps.historyCallback({ chat: key.chat, thread: key.thread, limit, prefetch: true }),
|
|
261
270
|
now,
|
|
262
271
|
})
|
|
263
272
|
}
|
|
@@ -549,18 +558,6 @@ function describe(err: unknown): string {
|
|
|
549
558
|
return err instanceof Error ? err.message : String(err)
|
|
550
559
|
}
|
|
551
560
|
|
|
552
|
-
async function raceWithTimeout<T>(work: Promise<T>, ms: number, label: string): Promise<T> {
|
|
553
|
-
let timer: ReturnType<typeof setTimeout> | null = null
|
|
554
|
-
const timeout = new Promise<never>((_, reject) => {
|
|
555
|
-
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms)
|
|
556
|
-
})
|
|
557
|
-
try {
|
|
558
|
-
return await Promise.race([work, timeout])
|
|
559
|
-
} finally {
|
|
560
|
-
if (timer !== null) clearTimeout(timer)
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
|
|
564
561
|
function dropHint(reason: InboundDropReason): string {
|
|
565
562
|
switch (reason) {
|
|
566
563
|
case 'empty_content':
|