typeclaw 0.39.4 → 0.39.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "typeclaw",
3
- "version": "0.39.4",
3
+ "version": "0.39.6",
4
4
  "homepage": "https://github.com/typeclaw/typeclaw#readme",
5
5
  "bugs": {
6
6
  "url": "https://github.com/typeclaw/typeclaw/issues"
@@ -715,11 +715,20 @@ export function detectAttentionEscalation(text: string, prior?: QuestionSignal |
715
715
  // unconditionally and gives OpenAI models maximum effort on escalation turns.
716
716
  const ESCALATED_LEVEL: ThinkingLevel = 'xhigh'
717
717
 
718
+ // `allowEscalation: false` pins the turn to `sessionDefault`. Subagent turns set
719
+ // it because their "user prompt" is composed by another agent, not a human — so
720
+ // the question-mark/frustration heuristics would misread a multi-question brief
721
+ // and silently bump a `fast`/`low` worker to `xhigh`. Default `true` keeps the
722
+ // human-facing sessions (TUI, channels, cron) escalating as before.
723
+ export type TurnThinkingOptions = { allowEscalation?: boolean }
724
+
718
725
  export function resolveTurnThinkingLevel(
719
726
  text: string,
720
727
  sessionDefault: ThinkingLevel | undefined,
721
728
  prior?: QuestionSignal | null,
729
+ options?: TurnThinkingOptions,
722
730
  ): ThinkingLevel | undefined {
731
+ if (options?.allowEscalation === false) return sessionDefault
723
732
  return detectAttentionEscalation(text, prior) ? ESCALATED_LEVEL : sessionDefault
724
733
  }
725
734
 
@@ -735,7 +744,8 @@ export function applyTurnThinkingLevel(
735
744
  text: string,
736
745
  sessionDefault: ThinkingLevel | undefined,
737
746
  prior?: QuestionSignal | null,
747
+ options?: TurnThinkingOptions,
738
748
  ): void {
739
- const resolved = resolveTurnThinkingLevel(text, sessionDefault, prior)
749
+ const resolved = resolveTurnThinkingLevel(text, sessionDefault, prior, options)
740
750
  if (resolved !== undefined) session.setThinkingLevel(resolved)
741
751
  }
@@ -338,7 +338,9 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
338
338
  retrievalContext.results.length > 0
339
339
  ? `${renderTurnTimeAnchor()}\n\n${userPromptForTurn}\n\n${retrievalContext.results}`
340
340
  : `${renderTurnTimeAnchor()}\n\n${userPromptForTurn}`
341
- applyTurnThinkingLevel(session, userPromptForTurn, session.thinkingLevel)
341
+ applyTurnThinkingLevel(session, userPromptForTurn, session.thinkingLevel, undefined, {
342
+ allowEscalation: false,
343
+ })
342
344
  const result = await promptPersistentTurnWithFallback({
343
345
  refs: resolveFallbackChain(getConfig().models, resolveSubagentProfile(subagent, sessionOptions)),
344
346
  currentModelRef: activeModelRef,
@@ -353,7 +355,9 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
353
355
  activeModelRef = ref
354
356
  },
355
357
  beforeAttempt: () => {
356
- applyTurnThinkingLevel(session, userPromptForTurn, session.thinkingLevel)
358
+ applyTurnThinkingLevel(session, userPromptForTurn, session.thinkingLevel, undefined, {
359
+ allowEscalation: false,
360
+ })
357
361
  },
358
362
  onAttemptFailed: (attempt) => {
359
363
  console.warn(
@@ -92,9 +92,9 @@ export function createChannelReplyTool({
92
92
  resolve_review_thread: Type.Optional(
93
93
  Type.Boolean({
94
94
  description:
95
- 'GitHub review threads ONLY — ignored on Slack, Discord, Telegram, KakaoTalk, and any non-github session, and ignored on a github reply that is not inside a `thread`. On those, leave this unset and ignore the rest of this description. ' +
96
- 'On a github reply inside a review thread you authored: when your `text` acknowledges the concern is fixed/verified/addressed (e.g. "verified at <sha>", "thanks, that resolves it"), treat setting this `true` as the expected close-out — do it in the SAME call. This is a strong instruction, not a schema requirement: the field stays optional and nothing rejects an acknowledgement that omits it, but a bare ack without it leaves the thread open, because a successful reply ends the turn and the resolve cannot run in a later one. So this flag is the only way the close-out actually happens. ' +
97
- "It is safe to set by default: the runtime resolves BEFORE posting and ONLY if the thread's root comment is yours — it refuses (and blocks the reply) on a human reviewer's thread, so you never close someone else's open question. You need not pre-check authorship; just set it on your acknowledgement and let the runtime enforce ownership. Leave it unset when you intend to keep the thread open (partial fix, disagreement, mid-discussion).",
95
+ 'GitHub PR review threads ONLY. On a TERMINAL (`continue:false`) github PR review-thread reply that carries `text`, this is REQUIRED: you must set it to `true` or `false` and omitting it is rejected — you will be told to re-call with an explicit choice. (It stays a normal optional field everywhere else: ignored on Slack/Discord/Telegram/KakaoTalk and any non-github session, on github replies outside a review thread, on attachments-only replies, and on mid-turn `continue:true` status updates — leave it unset there.) ' +
96
+ 'Set `true` when your `text` acknowledges the concern is fixed/verified/addressed (e.g. "verified at <sha>", "thanks, that resolves it"): the runtime resolves the thread BEFORE posting, so the close-out actually happens in this same turn — a successful reply ends the turn, so a resolve deferred to "later" never runs. ' +
97
+ "Set `false` when the thread should stay open (partial fix, disagreement, mid-discussion). It is safe to set `true` by default: the runtime resolves ONLY if the thread's root comment is yours — it refuses (and blocks the reply) on a human reviewer's thread, so you never close someone else's open question. You need not pre-check authorship; let the runtime enforce ownership.",
98
98
  }),
99
99
  ),
100
100
  }),
@@ -140,6 +140,27 @@ export function createChannelReplyTool({
140
140
  }
141
141
  }
142
142
 
143
+ // Required-choice guard: a terminal github review-thread text reply must
144
+ // make an explicit resolve_review_thread choice. The model kept silently
145
+ // omitting the flag after acknowledging a fix, leaving the thread open;
146
+ // forcing the choice (the discipline `continue` already enforces) closes
147
+ // the loop where prose instructions did not. The field stays optional in
148
+ // the schema precisely so omission is still detectable here (vs explicit
149
+ // false). Runs before the resolve/guards so a missing choice never acts.
150
+ const missingResolveChoice = missingReviewThreadResolveChoiceError({
151
+ origin,
152
+ text,
153
+ isContinue: keepTurnAlive,
154
+ resolveReviewThread: params.resolve_review_thread,
155
+ })
156
+ if (missingResolveChoice) {
157
+ logger.warn(formatChannelToolFailure('channel_reply', missingResolveChoice))
158
+ return {
159
+ content: [{ type: 'text' as const, text: `channel_reply denied: ${missingResolveChoice}` }],
160
+ details: { ok: false, error: missingResolveChoice },
161
+ }
162
+ }
163
+
143
164
  // False-receipt guard: deny a terminal reply that CLAIMS a PR verdict /
144
165
  // thread close-out the agent never actually performed this turn. Warn-tier
145
166
  // claims fall through and have their notice appended on success below.
@@ -279,6 +300,30 @@ export function createChannelReplyTool({
279
300
  })
280
301
  }
281
302
 
303
+ // Returns the denial string when a terminal github PR review-thread text reply
304
+ // omits an explicit resolve_review_thread choice, or '' when no choice is owed.
305
+ // Scoped by `^pr:\d+$` (not just thread !== null) so a future threaded github
306
+ // context can't be forced into a resolve choice that means nothing there.
307
+ function missingReviewThreadResolveChoiceError(input: {
308
+ origin: ChannelReplyOrigin
309
+ text: string | undefined
310
+ isContinue: boolean
311
+ resolveReviewThread: boolean | undefined
312
+ }): string {
313
+ const isGithubPrReviewThread =
314
+ input.origin.adapter === 'github' && /^pr:\d+$/.test(input.origin.chat) && input.origin.thread !== null
315
+ const hasText = input.text !== undefined && input.text.trim() !== ''
316
+ if (!isGithubPrReviewThread || !hasText || input.isContinue || input.resolveReviewThread !== undefined) {
317
+ return ''
318
+ }
319
+ return (
320
+ 'This is a terminal github PR review-thread reply with text, so `resolve_review_thread` is required: ' +
321
+ 'set it to `true` when the concern is fixed and this bot-authored thread should be closed (the runtime ' +
322
+ 'resolves before posting and only on your own thread), or `false` to leave it open. You omitted it; ' +
323
+ 're-call channel_reply with an explicit boolean.'
324
+ )
325
+ }
326
+
282
327
  // Returns an error string when the resolve should block the reply, or null
283
328
  // when it's safe to proceed. Only `no-match` (the thread is already gone, so
284
329
  // there's nothing to close) joins success as non-blocking; every hard failure
@@ -13,23 +13,58 @@
13
13
  // for the full rationale and AGENTS.md §"Web search" for the original story.
14
14
 
15
15
  import { curlImpersonate } from './curl-impersonate'
16
+ import { createKeyedSemaphore } from './keyed-semaphore'
17
+ import { type SearchRetryOptions, withSearchRetry } from './search-retry'
16
18
 
17
19
  export { _setCurlBinaryForTest } from './curl-impersonate'
18
20
 
19
21
  const DDG_LITE_URL = 'https://lite.duckduckgo.com/lite/'
20
22
 
23
+ // Empirically derived: a clean source IP serves DDG-lite at concurrency ~4 but
24
+ // trips a sticky multi-minute CAPTCHA at 5-6 (see keyed-semaphore.ts header).
25
+ // The whole agent egresses from one IP and the scout subagent fans out parallel
26
+ // searches, so 2 sits well under the observed ceiling with headroom for the
27
+ // concurrent membership-style overlap — matching the webex-prefetch-limiter
28
+ // default for the same reason.
29
+ export const DDG_CONCURRENCY = 2
30
+
31
+ // Process-wide, so the cap spans every session/subagent/cron in this agent.
32
+ const searchLimiter = createKeyedSemaphore({ concurrency: DDG_CONCURRENCY })
33
+
34
+ const DDG_KEY = 'ddg'
35
+
21
36
  export type DdgResult = {
22
37
  title: string
23
38
  url: string
24
39
  snippet: string
25
40
  }
26
41
 
42
+ type RetryOverride = Pick<SearchRetryOptions, 'attempts' | 'baseDelayMs' | 'maxDelayMs' | 'sleep'>
43
+ let retryOverride: RetryOverride = {}
44
+
45
+ // Test-only seam: lets *.test.ts collapse the generous production backoff to
46
+ // zero-delay single-shot retries so CAPTCHA tests don't burn real seconds.
47
+ // Production never calls this — the empty default above is what production sees.
48
+ export function _setSearchRetryForTest(override: RetryOverride | null): void {
49
+ retryOverride = override ?? {}
50
+ }
51
+
27
52
  export async function ddgSearch(query: string, limit: number, signal?: AbortSignal): Promise<DdgResult[]> {
28
- const html = await fetchDdgHtml(query, signal)
29
- if (isCaptcha(html)) {
30
- throw new DdgCaptchaError()
31
- }
32
- return parseDdgHtml(html).slice(0, limit)
53
+ return searchLimiter.run(
54
+ DDG_KEY,
55
+ () =>
56
+ withSearchRetry(
57
+ async () => {
58
+ const html = await fetchDdgHtml(query, signal)
59
+ if (isCaptcha(html)) {
60
+ throw new DdgCaptchaError()
61
+ }
62
+ return parseDdgHtml(html).slice(0, limit)
63
+ },
64
+ { ...retryOverride, shouldRetry: (error) => error instanceof DdgCaptchaError, signal },
65
+ ),
66
+ signal,
67
+ )
33
68
  }
34
69
 
35
70
  export class DdgCaptchaError extends Error {
@@ -0,0 +1,114 @@
1
+ // A process-wide keyed concurrency limiter: at most `concurrency` calls run
2
+ // at once PER KEY, and work queued behind a full key WAITS for a slot rather
3
+ // than being skipped. Distinct keys never contend.
4
+ //
5
+ // This is the queue-and-wait sibling of src/channels/adapters/webex-prefetch-
6
+ // limiter.ts. That limiter uses admit-or-skip because its work (history
7
+ // prefetch) is an optional context improvement — dropping it is fine. Here the
8
+ // work is a search the model explicitly asked for, so dropping it is NOT fine;
9
+ // callers must block until a slot frees.
10
+ //
11
+ // Why this exists: one typeclaw agent is one process, and all sessions
12
+ // (TUI, channel, cron, subagent) share that process. A search engine like
13
+ // DuckDuckGo rate-limits by source IP, and the whole agent egresses from one
14
+ // IP, so parallel searches across sessions/subagents stack onto the same
15
+ // per-engine budget. An empirical probe of lite.duckduckgo.com showed a clean
16
+ // IP serves ~4 concurrent requests but trips a sticky multi-minute CAPTCHA at
17
+ // 5-6 — and once tripped, even serial requests stay boxed. Capping concurrency
18
+ // PER ENGINE keeps the burst under that ceiling so the box is never tripped.
19
+
20
+ export type KeyedSemaphore = {
21
+ // Runs `work` once a slot for `key` is free, waiting if necessary. The slot
22
+ // is released when `work` settles (resolve or reject). Distinct keys run
23
+ // fully independently. If `signal` aborts while the call is still queued (or
24
+ // is already aborted on entry), the call rejects with SemaphoreAbortedError
25
+ // WITHOUT ever running `work` or holding a slot.
26
+ run<T>(key: string, work: () => Promise<T>, signal?: AbortSignal): Promise<T>
27
+ }
28
+
29
+ export type KeyedSemaphoreOptions = {
30
+ // Max concurrent runs PER KEY. Defaults to 1.
31
+ concurrency?: number
32
+ }
33
+
34
+ export class SemaphoreAbortedError extends Error {
35
+ constructor() {
36
+ super('Semaphore acquisition aborted before admission.')
37
+ this.name = 'SemaphoreAbortedError'
38
+ }
39
+ }
40
+
41
+ export function createKeyedSemaphore(options: KeyedSemaphoreOptions = {}): KeyedSemaphore {
42
+ const concurrency = Math.max(1, Math.floor(options.concurrency ?? 1))
43
+
44
+ type Waiter = () => void
45
+ type Pool = { active: number; waiters: Waiter[] }
46
+ const pools = new Map<string, Pool>()
47
+
48
+ const poolFor = (key: string): Pool => {
49
+ let pool = pools.get(key)
50
+ if (pool === undefined) {
51
+ pool = { active: 0, waiters: [] }
52
+ pools.set(key, pool)
53
+ }
54
+ return pool
55
+ }
56
+
57
+ const release = (key: string): void => {
58
+ const pool = pools.get(key)
59
+ if (pool === undefined) return
60
+ pool.active--
61
+ const next = pool.waiters.shift()
62
+ if (next) {
63
+ pool.active++
64
+ next()
65
+ // Drop idle pools so a long-lived semaphore does not accumulate one entry
66
+ // per key ever seen.
67
+ } else if (pool.active === 0 && pool.waiters.length === 0) {
68
+ pools.delete(key)
69
+ }
70
+ }
71
+
72
+ const acquire = (key: string, signal?: AbortSignal): Promise<void> => {
73
+ if (signal?.aborted) return Promise.reject(new SemaphoreAbortedError())
74
+ const pool = poolFor(key)
75
+ if (pool.active < concurrency) {
76
+ pool.active++
77
+ return Promise.resolve()
78
+ }
79
+ return new Promise<void>((resolve, reject) => {
80
+ let settled = false
81
+ const grant: Waiter = () => {
82
+ if (settled) {
83
+ // Granted after we already aborted: hand the freed slot to the next
84
+ // waiter so an aborted grant never leaks a permanently-held slot.
85
+ release(key)
86
+ return
87
+ }
88
+ settled = true
89
+ signal?.removeEventListener('abort', onAbort)
90
+ resolve()
91
+ }
92
+ const onAbort = (): void => {
93
+ if (settled) return
94
+ settled = true
95
+ const idx = pool.waiters.indexOf(grant)
96
+ if (idx >= 0) pool.waiters.splice(idx, 1)
97
+ reject(new SemaphoreAbortedError())
98
+ }
99
+ pool.waiters.push(grant)
100
+ signal?.addEventListener('abort', onAbort, { once: true })
101
+ })
102
+ }
103
+
104
+ return {
105
+ async run<T>(key: string, work: () => Promise<T>, signal?: AbortSignal): Promise<T> {
106
+ await acquire(key, signal)
107
+ try {
108
+ return await work()
109
+ } finally {
110
+ release(key)
111
+ }
112
+ },
113
+ }
114
+ }
@@ -0,0 +1,82 @@
1
+ // Retry-with-backoff for search-engine calls that hit a transient rate-limit
2
+ // or CAPTCHA wall.
3
+ //
4
+ // Empirically, lite.duckduckgo.com does NOT recover in milliseconds once it
5
+ // starts gating: a tripped source IP stays boxed for minutes, and the gate
6
+ // surfaces as a CAPTCHA page (parsed as DdgCaptchaError) rather than a clean
7
+ // 429. So the backoff here is deliberately generous (seconds, growing) — a
8
+ // tight millisecond retry would just burn the remaining budget and confirm the
9
+ // box. The concurrency cap (keyed-semaphore.ts) is the real preventer; this is
10
+ // the second line of defense for a request that still raced into a gate.
11
+ //
12
+ // `shouldRetry` decides which errors are worth retrying — only transient
13
+ // rate-limit/CAPTCHA signals, never a parse error or a hard network failure.
14
+
15
+ export type SearchRetryOptions = {
16
+ attempts?: number
17
+ baseDelayMs?: number
18
+ maxDelayMs?: number
19
+ shouldRetry: (error: unknown) => boolean
20
+ signal?: AbortSignal
21
+ // Injectable sleep so tests can run without real timers.
22
+ sleep?: (ms: number, signal?: AbortSignal) => Promise<void>
23
+ }
24
+
25
+ export const DEFAULT_SEARCH_RETRY_ATTEMPTS = 3
26
+ export const DEFAULT_SEARCH_RETRY_BASE_DELAY_MS = 2_000
27
+ export const DEFAULT_SEARCH_RETRY_MAX_DELAY_MS = 15_000
28
+
29
+ export async function withSearchRetry<T>(work: () => Promise<T>, options: SearchRetryOptions): Promise<T> {
30
+ const attempts = Math.max(1, Math.floor(options.attempts ?? DEFAULT_SEARCH_RETRY_ATTEMPTS))
31
+ const baseDelayMs = options.baseDelayMs ?? DEFAULT_SEARCH_RETRY_BASE_DELAY_MS
32
+ const maxDelayMs = options.maxDelayMs ?? DEFAULT_SEARCH_RETRY_MAX_DELAY_MS
33
+ const sleep = options.sleep ?? defaultSleep
34
+
35
+ let lastError: unknown
36
+ for (let attempt = 0; attempt < attempts; attempt++) {
37
+ if (options.signal?.aborted) throw new SearchRetryAbortedError()
38
+ try {
39
+ return await work()
40
+ } catch (error) {
41
+ lastError = error
42
+ const isLastAttempt = attempt === attempts - 1
43
+ if (isLastAttempt || !options.shouldRetry(error)) throw error
44
+ await sleep(backoffDelayMs(attempt, baseDelayMs, maxDelayMs), options.signal)
45
+ }
46
+ }
47
+ throw lastError
48
+ }
49
+
50
+ export class SearchRetryAbortedError extends Error {
51
+ constructor() {
52
+ super('Search retry aborted.')
53
+ this.name = 'SearchRetryAbortedError'
54
+ }
55
+ }
56
+
57
+ // Exponential backoff with full jitter (AWS "Exponential Backoff And Jitter").
58
+ // Full jitter — random in [0, capped] — decorrelates concurrent retriers so a
59
+ // burst that all tripped the gate together does not re-collide on the same
60
+ // wake-up, which would just re-trip the box.
61
+ export function backoffDelayMs(attempt: number, baseDelayMs: number, maxDelayMs: number): number {
62
+ const capped = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt)
63
+ return Math.floor(Math.random() * capped)
64
+ }
65
+
66
+ function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
67
+ return new Promise((resolve, reject) => {
68
+ if (signal?.aborted) {
69
+ reject(new SearchRetryAbortedError())
70
+ return
71
+ }
72
+ const timer = setTimeout(() => {
73
+ signal?.removeEventListener('abort', onAbort)
74
+ resolve()
75
+ }, ms)
76
+ const onAbort = (): void => {
77
+ clearTimeout(timer)
78
+ reject(new SearchRetryAbortedError())
79
+ }
80
+ signal?.addEventListener('abort', onAbort, { once: true })
81
+ })
82
+ }
@@ -77,6 +77,10 @@ End every response with this exact structure:
77
77
  - If the question requires EXTERNAL/web information (docs, library reference, web search, fetching a URL), say so explicitly and tell the caller to spawn \`scout\` instead. Do not try to answer external questions from memory.
78
78
  - If you cannot find what was asked, say so explicitly with what you DID find and what surfaces you searched.`
79
79
 
80
+ // `profile` is dropped, not passed through: explorer is a fast-tier specialist, so
81
+ // a parent must not bump it off `fast` via a per-spawn `profile` override (which the
82
+ // `spawn_subagent` tool injects into the payload and `profileFromPayload` would
83
+ // otherwise honor). Other fields still pass through untouched.
80
84
  export const explorerPayloadSchema = z
81
85
  .object({
82
86
  requestId: z.string().optional(),
@@ -84,6 +88,7 @@ export const explorerPayloadSchema = z
84
88
  description: z.string().optional(),
85
89
  })
86
90
  .passthrough()
91
+ .transform(({ profile: _profile, ...rest }) => rest)
87
92
 
88
93
  export type ExplorerPayload = z.infer<typeof explorerPayloadSchema>
89
94
 
@@ -135,7 +135,7 @@ A `[dreaming] citation-superset violation: …` warning logs the dropped ids and
135
135
  - **`memory/streams/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`. Force-committed alongside the shards. A `fragment` event carries optional **situational provenance** for "who said what, where, and when": `who` (display name/handle of the attributed speaker, LLM-supplied per-fragment by `memory-logger` only when one transcript speaker clearly owns the evidence), `where` (`{ adapter, workspace, workspaceName?, chat, chatName?, thread }` — runtime-stamped from the session origin in the append tool, channel origins only), and `when` (reuses the existing `ts`). Both are optional, so legacy fragments and non-channel (TUI) origins parse with them absent. `who`/`where` are deliberately NOT in `fragmentContentHash` (dedup stays content-identity) and NOT in `fragmentEmbeddableText` (provenance is retrieval metadata, not embedded — it must not burn the 512-token budget or let same-room/same-person dominate recall). **Secret-scanning covers provenance too**: `who` (LLM-supplied) clears the same `detectSecrets` guard as `topic`/`body` and a credential-shaped value is refused with the retryable error; the origin-derived `where` names (`workspaceName`/`chatName`) are external channel data the agent can't rewrite, so a credential-shaped name is silently dropped (raw id kept) rather than wedging the logger. Raw ids are platform-issued identifiers, not free text, so they are not scanned. `where` is runtime-stamped rather than LLM-supplied because the channel is stable for a whole logger run; `who` is per-fragment because a single run spans many speakers and `origin.lastInboundAuthorId` is only the spawn-time one — auto-stamping it would misattribute fragments about earlier speakers. Surfaced to the agent (not embedded) via `memory_search` (`StreamMatch.who/when/where`), `hybridSearch` (`HybridSearchResult.who/when/where` on self-resolved undreamed fragments), and the per-turn retrieval render (`renderProvenanceLine` → `_Jisoo in #incidents on 2026-06-24_`).
136
136
  - **`memory/MEMORY.md.pre-shard.bak`** — legacy pre-shard backup left by older TypeClaw versions. Safe to delete after verifying.
137
137
  - **`memory/skills/<name>/SKILL.md`** — muscle memory (Form A). Skills the dreaming subagent distills from repeated procedures. Auto-loaded as first-class skills. CLI/plugin suggestions (Forms B and C) are recorded as topic shards instead — see [Muscle memory](#muscle-memory-three-forms).
138
- - **`memory/references/<slug>.md`** — verbatim reference artifacts (gated behind `memory.references.enabled`). Written by `store_reference`. When vector is enabled, a freshly stored reference is embedded into the vector index immediately via the `store_reference` on-write hook (`vector/reference-index-on-write.ts`), mirroring the stream-fragment on-write hook — so it is vector-retrievable within the same container uptime instead of only after the next startup index build. Chunks are derived by the same `referencePassagesForOne` the startup build uses, and re-storing a shorter body prunes the stale higher-index chunk rows.
138
+ - **`memory/references/<slug>.md`** — verbatim reference artifacts. Written by `store_reference`, a `memory-logger` custom tool that is always registered (there is no `references.enabled` config gate — references are unconditionally available; they simply stay empty until the `memory-logger` chooses to store one). A fragment may cite a stored reference via `append`'s `references: ['<slug>']`, but the append tool validates each slug against the references that actually exist on disk and silently drops the rest (`resolveValidReferences`), so a hallucinated slug never lands a dangling citation in the stream. When vector is enabled, a freshly stored reference is embedded into the vector index immediately via the `store_reference` on-write hook (`vector/reference-index-on-write.ts`), mirroring the stream-fragment on-write hook — so it is vector-retrievable within the same container uptime instead of only after the next startup index build. Chunks are derived by the same `referencePassagesForOne` the startup build uses, and re-storing a shorter body prunes the stale higher-index chunk rows.
139
139
  - **`memory/.dreaming-state.json`** — per-day dreamed-id sets.
140
140
  - **`memory/.retrieval-cache/<sessionId>.md`** — ephemeral retrieval summaries. Written by `memory-retrieval`, read by `loadMemory` on the next prompt of the same session, unlinked on `session.end`.
141
141
 
@@ -145,7 +145,7 @@ The plugin emits structured `[plugin:memory]` log lines (no separate metrics inf
145
145
 
146
146
  - **`[memory-logger] <session> done fragments_written=N elapsed_ms=…`** — how many fragments a logger run captured (delta of fragment events in today's stream).
147
147
  - **`[dreaming] done topics_created=N topics_removed=N superseded_new=N fragments_dropped=N over_budget=N elapsed_ms=…`** — consolidation activity per run: new/removed topic shards (by snapshot path diff), net citations moved into `superseded:`, fragments GC'd, and `over_budget` (the count of topic shards whose embeddable text exceeds the model token budget — see [Embedding token budget](#embedding-token-budget)). `topics_created` + reinforcement is the "is memory getting sharper" signal; `superseded_new` is contradiction-edit volume; a persistently non-zero `over_budget` means the subagent isn't compacting the flagged shards. Derived by `computeDreamingMetrics` from the pre/post shard snapshots.
148
- - **`[vector-retrieval] mode=index topic_results=N stream_results=N reference_results=N elapsed_ms=… [suppressed=1]`** (or `mode=direct topics=N`) — per-turn retrieval breakdown. `stream_results` counts undreamed-fragment hits that self-resolved (no citing topic yet) — the freshness-window usage signal that informs whether the undreamed surface earns its keep. `reference_results` counts verbatim-reference hits (only non-zero when `references.enabled`), broken out from `stream_results` so the two surfaces are distinguishable. `elapsed_ms` times the full `hybridSearch` (embed + cosine scan + keyword lane + RRF fusion) — the on-critical-path retrieval cost. `suppressed=1` appears when the query matched nothing after the relevance gate, so an empty memory block is distinguishable from a missing one.
148
+ - **`[vector-retrieval] mode=index topic_results=N stream_results=N reference_results=N elapsed_ms=… [suppressed=1]`** (or `mode=direct topics=N`) — per-turn retrieval breakdown. `stream_results` counts undreamed-fragment hits that self-resolved (no citing topic yet) — the freshness-window usage signal that informs whether the undreamed surface earns its keep. `reference_results` counts verbatim-reference hits (non-zero once the `memory-logger` has stored at least one reference and vector is enabled), broken out from `stream_results` so the two surfaces are distinguishable. `elapsed_ms` times the full `hybridSearch` (embed + cosine scan + keyword lane + RRF fusion) — the on-critical-path retrieval cost. `suppressed=1` appears when the query matched nothing after the relevance gate, so an empty memory block is distinguishable from a missing one.
149
149
 
150
150
  These are intentionally verbose for now so behavior is observable in logs; trim once the useful subset is known.
151
151
 
@@ -21,9 +21,17 @@ export type FragmentsAppendedHook = (fragments: FragmentEvent[], context: Fragme
21
21
  // value) because the same factory builds tools whose origin is only known later.
22
22
  export type OriginProvider = () => SessionOrigin | undefined
23
23
 
24
+ // Returns the set of reference slugs the agent is allowed to cite from
25
+ // `references[]`: every reference that exists on disk (a `store_reference` call
26
+ // writes the file synchronously before returning the slug, so a same-run store
27
+ // is already on disk by append time). Lets the append tool strip hallucinated
28
+ // slugs the model invents without ever calling `store_reference`.
29
+ export type ReferenceSlugResolver = (agentDir: string) => Promise<Iterable<string>>
30
+
24
31
  export type CreateAppendToolOptions = {
25
32
  onFragmentsAppended?: FragmentsAppendedHook
26
33
  originProvider?: OriginProvider
34
+ referenceSlugResolver?: ReferenceSlugResolver
27
35
  }
28
36
 
29
37
  // Stamped fields are the stable channel coordinates only. Author identity is
@@ -57,10 +65,10 @@ function redactIfSecret(name: string | undefined): string | undefined {
57
65
  }
58
66
 
59
67
  export function createAppendTool(options: CreateAppendToolOptions = {}) {
60
- const { onFragmentsAppended, originProvider } = options
68
+ const { onFragmentsAppended, originProvider, referenceSlugResolver } = options
61
69
  return defineTool({
62
70
  description:
63
- "Append a memory fragment to today's JSONL daily stream and advance the watermark. The runtime serializes your call into a JSON line and chooses the filename — do not emit raw JSON and do not pass a path. `topic`/`body` are the fragment's substance; `source` is the parent session id; `entry` is the transcript-entry-id this fragment anchors to; `latestEntryId` is the latest transcript-entry-id you evaluated in this run (advances the watermark, may equal `entry` or be later). `who` is the display name/handle of the person the fragment's evidence is attributable to — set it ONLY when one transcript speaker clearly owns the evidence; omit it when the fact is the user's own, spans multiple speakers, or is not attributable. The channel/room/platform (`where`) is stamped automatically from the session origin — do not pass it and do not restate it in the body. Refuses content with recognized credential patterns and refuses byte-equivalent topic+body within the same daily stream.",
71
+ "Append a memory fragment to today's JSONL daily stream and advance the watermark. The runtime serializes your call into a JSON line and chooses the filename — do not emit raw JSON and do not pass a path. `topic`/`body` are the fragment's substance; `source` is the parent session id; `entry` is the transcript-entry-id this fragment anchors to; `latestEntryId` is the latest transcript-entry-id you evaluated in this run (advances the watermark, may equal `entry` or be later). `references` may ONLY contain slugs returned by `store_reference` (it returns the slug it wrote) — never topic ids, PR names, stream paths, or invented labels; unknown slugs are dropped. `who` is the display name/handle of the person the fragment's evidence is attributable to — set it ONLY when one transcript speaker clearly owns the evidence; omit it when the fact is the user's own, spans multiple speakers, or is not attributable. The channel/room/platform (`where`) is stamped automatically from the session origin — do not pass it and do not restate it in the body. Refuses content with recognized credential patterns and refuses byte-equivalent topic+body within the same daily stream.",
64
72
  parameters: z.object({
65
73
  topic: z.string().min(1),
66
74
  body: z.string().min(1),
@@ -104,8 +112,9 @@ export function createAppendTool(options: CreateAppendToolOptions = {}) {
104
112
  topic,
105
113
  body,
106
114
  }
107
- if (references !== undefined && references.length > 0) {
108
- fragment.references = references
115
+ const validReferences = await resolveValidReferences(references, referenceSlugResolver, ctx)
116
+ if (validReferences.length > 0) {
117
+ fragment.references = validReferences
109
118
  }
110
119
  if (who !== undefined) fragment.who = who
111
120
  if (where !== undefined) fragment.where = where
@@ -173,6 +182,35 @@ function dailyStreamPath(agentDir: string): string {
173
182
  return streamFilePath(agentDir, formatLocalDate())
174
183
  }
175
184
 
185
+ // The model treats `references[]` as a free-text related-id field, citing empty
186
+ // strings, stream paths, and slugs for references it never stored. Every path
187
+ // first normalizes: trim, drop blank/whitespace-only entries, and de-duplicate
188
+ // — an empty or repeated citation is never meaningful regardless of wiring.
189
+ // With a resolver, only slugs backed by a real reference file then survive; the
190
+ // rest are dropped and logged so dangling citations never reach the stream.
191
+ // Without one (the standalone `appendTool` export, no reference subsystem
192
+ // wired) the normalized slugs pass through.
193
+ async function resolveValidReferences(
194
+ references: string[] | undefined,
195
+ resolver: ReferenceSlugResolver | undefined,
196
+ ctx: { agentDir: string; logger?: { warn(message: string): void } },
197
+ ): Promise<string[]> {
198
+ if (references === undefined || references.length === 0) return []
199
+ const requested = [...new Set(references.map((slug) => slug.trim()).filter((slug) => slug.length > 0))]
200
+ if (resolver === undefined) return requested
201
+
202
+ const known = new Set(await resolver(ctx.agentDir))
203
+ const valid = requested.filter((slug) => known.has(slug))
204
+ const dropped = requested.filter((slug) => !known.has(slug))
205
+ if (dropped.length > 0) {
206
+ ctx.logger?.warn(
207
+ `[memory] dropped ${dropped.length} unknown reference slug(s) from fragment: ${dropped.join(', ')}. ` +
208
+ `references[] may only cite slugs returned by store_reference.`,
209
+ )
210
+ }
211
+ return valid
212
+ }
213
+
176
214
  function assertNoSecrets(parts: ReadonlyArray<string | undefined>): void {
177
215
  const content = parts.filter((part): part is string => part !== undefined).join('\n')
178
216
  const secrets = detectSecrets(content)
@@ -7,6 +7,7 @@ import { formatLocalDate } from '@/shared'
7
7
  import { advanceWatermarkTool, createAppendTool, type FragmentsAppendedHook } from './append-tool'
8
8
  import { findEntryTool } from './find-entry-tool'
9
9
  import { streamFilePath, streamsDir } from './paths'
10
+ import { listReferenceSlugs } from './references/load-references'
10
11
  import { createStoreReferenceTool, type ReferenceStoredHook } from './references/store-reference-tool'
11
12
  import { readEvents } from './stream-io'
12
13
  import { readLatestWatermark } from './watermark'
@@ -146,9 +147,11 @@ Skip these anti-patterns:
146
147
 
147
148
  # Verbatim references (store_reference tool)
148
149
 
149
- When the user explicitly asks to remember a verbatim artifact — SQL, code, a runbook, pasted spec — call \`store_reference({ title, body, origin: 'episode', tags: [] })\` with the byte-for-byte body. Do not distill or summarize it. The tool returns a slug; include it in \`append\` as \`references: ['<slug>']\` when writing the session fragment.
150
+ Store a verbatim artifact whenever its memory value depends on the EXACT text — SQL, code blocks, runbooks, pasted specs, config snippets, API payloads, review comments, or command output the user will want reproduced byte-for-byte later. This is not limited to "please remember this": if you are about to write a fragment that summarizes such an artifact, store the artifact too so the exact text survives. Call \`store_reference({ title, body, origin: 'episode', tags: [] })\` with the byte-for-byte body. Do not distill or summarize the body.
150
151
 
151
- If the reference is the only durable content, still write a fragment (topic "verbatim reference stored") naming what was stored and citing the reference, so the reference is linked into the stream.
152
+ \`store_reference\` returns a slug (the file it wrote under \`memory/references/\`). The ONLY valid contents of \`append\`'s \`references\` field are slugs you received back from \`store_reference\` in THIS run. Never put a topic id, PR name/number, stream path, URL, or any invented label in \`references\` — those are not references and will be silently dropped. If you did not call \`store_reference\`, omit \`references\` entirely. To cross-link a related topic, name it in the body prose instead.
153
+
154
+ If a stored reference is the only durable content, still write a fragment (topic "verbatim reference stored") naming what was stored and citing the returned slug, so the reference is linked into the stream.
152
155
 
153
156
  References are for artifacts whose exact text matters. A distilled memory fragment should name what the artifact is, who/what it applies to, and why it was retained, while the reference body holds the verbatim material. Do not paste large reference bodies into fragment text.
154
157
 
@@ -179,7 +182,7 @@ Recurrence is not duplication. A durable preference, pattern, workaround, or com
179
182
 
180
183
  # Fragment format
181
184
 
182
- Call \`append\` with \`{topic, body, source, entry, latestEntryId}\` or \`{topic, body, source, entry, latestEntryId, references}\`. The runtime serializes your call into the daily stream; you never write raw JSON. \`source\` is the parent session id. \`topic\` is a short noun phrase. \`entry\` is the specific transcript-entry-id that anchors this fragment's evidence. Each fragment carries its own entry id; do not stamp every fragment with the same latest evaluated id. \`latestEntryId\` is the latest entry evaluated in this run and advances the watermark. \`references\` is optional slugs from \`store_reference\`.
185
+ Call \`append\` with \`{topic, body, source, entry, latestEntryId}\` or \`{topic, body, source, entry, latestEntryId, references}\`. The runtime serializes your call into the daily stream; you never write raw JSON. \`source\` is the parent session id. \`topic\` is a short noun phrase. \`entry\` is the specific transcript-entry-id that anchors this fragment's evidence. Each fragment carries its own entry id; do not stamp every fragment with the same latest evaluated id. \`latestEntryId\` is the latest entry evaluated in this run and advances the watermark. \`references\` is optional and may ONLY contain slugs that \`store_reference\` returned to you in this run; anything else (topic ids, PR names, stream paths, invented labels) is dropped — omit the field if you stored nothing.
183
186
 
184
187
  **Situational provenance (\`who\` / \`where\`).** Memory is sharper when it remembers who said something and where. Two fields capture this:
185
188
 
@@ -297,6 +300,7 @@ export function createMemoryLoggerSubagent(
297
300
  const appendTool = createAppendTool({
298
301
  onFragmentsAppended: options.onFragmentsAppended,
299
302
  originProvider: () => currentOrigin,
303
+ referenceSlugResolver: (agentDir) => listReferenceSlugs(agentDir),
300
304
  })
301
305
  const storeReferenceTool = createStoreReferenceTool(options.onReferenceStored)
302
306
  const customTools = [findEntryTool, appendTool, storeReferenceTool, advanceWatermarkTool]
@@ -79,7 +79,9 @@ Your role is EXCLUSIVELY to analyze and report. The parent agent decides what to
79
79
 
80
80
  You run on a deliberately expensive model. Reading a sprawling file tree, a giant diff, or a pile of vendor docs into YOUR context burns that budget on grunt work. When a slice of the job is bulky-but-mechanical — "summarize what these 40 files do", "extract the public API of this module", "gather the relevant passages from this 2,000-line diff" — hand it to a cheaper worker with \`spawn_subagent\` and review the distilled result instead of the raw bulk.
81
81
 
82
- - Spawn read-only/research workers for context-heavy gathering, not for forming the verdict. The findings and the \`<review>\` block are YOURS — never delegate the judgment.
82
+ This is not a last resort — it is your DEFAULT for any non-trivial target, and it makes the review FASTER, not just leaner. Reach for \`scout\` and \`explorer\` first: \`explorer\` maps unfamiliar code and finds where things live, \`scout\` gathers focused facts from a known area. Decompose the target into independent gathering tasks and fan them out IN PARALLEL — emit several \`spawn_subagent\` calls in a SINGLE turn (they run concurrently and return together) rather than reading each slice yourself, one after another, into your own context. A wall-clock-faster review on a fresh, uncluttered context is the win; serial self-reading is the slow path you are explicitly steered away from.
83
+
84
+ - Spawn read-only/research workers (\`scout\`, \`explorer\`) for context-heavy gathering, not for forming the verdict. The findings and the \`<review>\` block are YOURS — never delegate the judgment.
83
85
  - Each delegated task must be self-contained: the worker does not see this conversation or the target. Put everything it needs in the prompt.
84
86
  - The chain is depth-limited: a worker you spawn cannot spawn again. Keep delegation one level deep.
85
87
  - \`subagent_output\`/\`subagent_cancel\` reach only the tasks YOU spawned. To gather in parallel, either emit all the independent \`spawn_subagent\` calls (sync, the default) in a SINGLE turn so they run concurrently and return together, or spawn them with \`run_in_background=true\` and fold each result in as its \`<system-reminder>\` arrives (your session stays alive until every child reports back). Either way, fold the results into your single review pass before you finish.
@@ -68,6 +68,10 @@ End every response with this exact structure:
68
68
  - If the question requires LOCAL information (codebase, files in /agent/, git history, memory), say so explicitly and tell the caller to spawn \`explorer\` instead.
69
69
  - If you cannot find what was asked, say so explicitly with what queries you tried and what you DID find.`
70
70
 
71
+ // `profile` is dropped, not passed through: scout is a fast-tier specialist, so a
72
+ // parent must not bump it off `fast` via a per-spawn `profile` override (which the
73
+ // `spawn_subagent` tool injects into the payload and `profileFromPayload` would
74
+ // otherwise honor). Other fields still pass through untouched.
71
75
  export const scoutPayloadSchema = z
72
76
  .object({
73
77
  requestId: z.string().optional(),
@@ -75,6 +79,7 @@ export const scoutPayloadSchema = z
75
79
  description: z.string().optional(),
76
80
  })
77
81
  .passthrough()
82
+ .transform(({ profile: _profile, ...rest }) => rest)
78
83
 
79
84
  export type ScoutPayload = z.infer<typeof scoutPayloadSchema>
80
85
 
@@ -13,6 +13,24 @@ export type OutboundEndpointKind =
13
13
  | 'issue-reaction'
14
14
  | 'pr-review-comment-reaction'
15
15
 
16
+ // The required-permissions checklist shown verbatim during interactive GitHub
17
+ // channel setup (both `typeclaw init` and `typeclaw channel add github`). Kept
18
+ // here — the single owner of GitHub permission copy — so the init and
19
+ // channel-add flows can't drift apart (they did once: Actions was added to one
20
+ // and not the other). The labels mirror github.com's UI strings so a user can
21
+ // grep their settings page for the exact text. `includeWebhookNote` appends the
22
+ // "TypeClaw will create and manage the repository webhooks for you" reassurance,
23
+ // shown only on the first-setup prompts (not the rotate/switch prompts).
24
+ export function githubRequiredPermissionsNote(includeWebhookNote: boolean): string {
25
+ const webhooks = includeWebhookNote
26
+ ? 'Webhooks read/write (TypeClaw will create and manage the repository webhooks for you).'
27
+ : 'Webhooks read/write.'
28
+ return [
29
+ 'Required permissions: Issues read/write, Pull requests read/write, Discussions read/write (if used),',
30
+ `Actions read/write, Metadata read, and ${webhooks}`,
31
+ ].join('\n')
32
+ }
33
+
16
34
  // Parses webhook-register errors of the shape `list hooks failed: <status> <body>`.
17
35
  // Returns the status code when it matches the two shapes GitHub emits for
18
36
  // missing access on the list-hooks endpoint: