pi-braid 0.43.40 → 0.43.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/CONTRIBUTING.md +1 -1
- package/TOOLS.md +10 -15
- package/build-info.json +4 -4
- package/dist/extensions/heartbeat/analytics/index.js +41 -18
- package/dist/extensions/memory/three-memory-stores/index.js +34 -38
- package/dist/index.d.ts +0 -2
- package/dist/index.js +0 -2
- package/dist/lib/tools/tool-families.d.ts +119 -0
- package/dist/lib/tools/tool-families.js +172 -0
- package/dist/loader/api.d.ts +2 -5
- package/dist/loader/api.js +7 -48
- package/dist/loader/loader.d.ts +0 -2
- package/dist/loader/loader.js +0 -4
- package/docs/LOCAL_DEV.md +1 -1
- package/docs/SETUP.md +1 -1
- package/package.json +1 -1
- package/skills/release-pipeline/SKILL.md +12 -0
- package/src/extensions/heartbeat/analytics/AGENTS.md +2 -2
- package/src/extensions/heartbeat/analytics/index.ts +50 -19
- package/src/extensions/memory/AGENTS.md +1 -2
- package/src/extensions/memory/three-memory-stores/index.ts +43 -41
- package/src/index.ts +0 -2
- package/src/lib/AGENTS.md +1 -0
- package/src/lib/tools/tool-families.ts +232 -0
- package/src/loader/AGENTS.md +4 -4
- package/src/loader/api.ts +9 -51
- package/src/loader/loader.ts +0 -5
package/dist/loader/api.js
CHANGED
|
@@ -17,11 +17,8 @@
|
|
|
17
17
|
* alone; argument completion and `list` are the places that enumerate what is
|
|
18
18
|
* accepted.
|
|
19
19
|
*
|
|
20
|
-
* The
|
|
21
|
-
*
|
|
22
|
-
* modules-reset, modules-list, modules-status. Each maps its whole argument
|
|
23
|
-
* onto the equivalent argv, so a script written against the old names keeps
|
|
24
|
-
* working until it is pointed at `/modules`.
|
|
20
|
+
* The list covers the core manifest only: a contrib extension is toggled
|
|
21
|
+
* through its `braid.contrib.<name>` key in settings, not from this command.
|
|
25
22
|
*/
|
|
26
23
|
import { getSettingsListTheme } from "@earendil-works/pi-coding-agent";
|
|
27
24
|
import { Container, SettingsList, } from "@earendil-works/pi-tui";
|
|
@@ -39,6 +36,8 @@ const VERBS = {
|
|
|
39
36
|
};
|
|
40
37
|
const USAGE = "Usage: /modules [on|off|toggle|reset|all|list] <ref>\n" +
|
|
41
38
|
"A ref is an extension id or a module group name. Tab completes both.";
|
|
39
|
+
/** Why a contrib extension is absent from this list, stated where it shows. */
|
|
40
|
+
const CONTRIB_NOTE = "Contrib extensions are toggled through braid.contrib in settings, not from this list.";
|
|
42
41
|
export default async function modulesApiExtension(pi) {
|
|
43
42
|
const run = async (argv, ctx) => {
|
|
44
43
|
const notify = (message, type = "info") => {
|
|
@@ -132,34 +131,7 @@ export default async function modulesApiExtension(pi) {
|
|
|
132
131
|
getArgumentCompletions: completeModules,
|
|
133
132
|
handler: (args, ctx) => run(splitArgs(args), ctx),
|
|
134
133
|
});
|
|
135
|
-
|
|
136
|
-
description: "Alias of /modules on <ref>. Retained for one release.",
|
|
137
|
-
getArgumentCompletions: completeRefs("on"),
|
|
138
|
-
handler: (args, ctx) => run([...forwarded("on", args)], ctx),
|
|
139
|
-
});
|
|
140
|
-
pi.registerCommand("modules-disable", {
|
|
141
|
-
description: "Alias of /modules off <ref>. Retained for one release.",
|
|
142
|
-
getArgumentCompletions: completeRefs("off"),
|
|
143
|
-
handler: (args, ctx) => run([...forwarded("off", args)], ctx),
|
|
144
|
-
});
|
|
145
|
-
pi.registerCommand("modules-toggle", {
|
|
146
|
-
description: "Alias of /modules toggle <ref>. Retained for one release.",
|
|
147
|
-
getArgumentCompletions: completeRefs("toggle"),
|
|
148
|
-
handler: (args, ctx) => run([...forwarded("toggle", args)], ctx),
|
|
149
|
-
});
|
|
150
|
-
pi.registerCommand("modules-reset", {
|
|
151
|
-
description: "Alias of /modules reset. Retained for one release.",
|
|
152
|
-
handler: async (_args, ctx) => run(["reset"], ctx),
|
|
153
|
-
});
|
|
154
|
-
pi.registerCommand("modules-list", {
|
|
155
|
-
description: "Alias of /modules list. Retained for one release.",
|
|
156
|
-
handler: async (_args, ctx) => run(["list"], ctx),
|
|
157
|
-
});
|
|
158
|
-
pi.registerCommand("modules-status", {
|
|
159
|
-
description: "Alias of /modules list. Retained for one release.",
|
|
160
|
-
handler: async (_args, ctx) => run(["list"], ctx),
|
|
161
|
-
});
|
|
162
|
-
log.info("Module API registered: /modules (+ six forwarding aliases)");
|
|
134
|
+
log.info("Module API registered: /modules");
|
|
163
135
|
}
|
|
164
136
|
// ── Argument parsing ────────────────────────────────────────────────────────
|
|
165
137
|
/** Split a command's argument string into argv; quoted runs stay one token. */
|
|
@@ -175,10 +147,6 @@ function splitArgs(args) {
|
|
|
175
147
|
}
|
|
176
148
|
return argv;
|
|
177
149
|
}
|
|
178
|
-
/** The argv an old name forwards: its whole argument is the ref. */
|
|
179
|
-
function forwarded(verb, args) {
|
|
180
|
-
return [verb, (args ?? "").trim()];
|
|
181
|
-
}
|
|
182
150
|
/** Why a reference was refused, naming the rule instead of listing refs. */
|
|
183
151
|
function refusal(ref, verb) {
|
|
184
152
|
if (!getModuleRefs().includes(ref)) {
|
|
@@ -212,15 +180,6 @@ function completeModules(prefix) {
|
|
|
212
180
|
}
|
|
213
181
|
return null;
|
|
214
182
|
}
|
|
215
|
-
/** Completion for an old name: the whole argument text is the reference. */
|
|
216
|
-
function completeRefs(verb) {
|
|
217
|
-
return (prefix) => {
|
|
218
|
-
const typed = (prefix ?? "").toLowerCase();
|
|
219
|
-
return getModuleRefs()
|
|
220
|
-
.filter((ref) => ref.startsWith(typed))
|
|
221
|
-
.map((ref) => ({ value: ref, label: ref, description: `Forwarded to /modules ${verb}` }));
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
183
|
function completeValues(prefix, values) {
|
|
225
184
|
const typed = (prefix ?? "").toLowerCase();
|
|
226
185
|
return values
|
|
@@ -248,7 +207,7 @@ function describe(desc, origin, enabled, loaded) {
|
|
|
248
207
|
/** The rows as text, for RPC mode and the `list` verb. */
|
|
249
208
|
function renderRows() {
|
|
250
209
|
const { rows, source } = moduleRows();
|
|
251
|
-
const lines = [`Extensions (${rows.length}, config: ${source})
|
|
210
|
+
const lines = [`Extensions (${rows.length}, config: ${source}):`, CONTRIB_NOTE];
|
|
252
211
|
const width = Math.min(28, Math.max(...rows.map((row) => row.id.length)));
|
|
253
212
|
for (const row of rows) {
|
|
254
213
|
lines.push(` ${row.value.padEnd(3)} ${row.id.padEnd(width)} ${row.description}`);
|
|
@@ -266,7 +225,7 @@ async function openModuleList(ctx) {
|
|
|
266
225
|
const container = new Container();
|
|
267
226
|
container.addChild(new (class {
|
|
268
227
|
render(_width) {
|
|
269
|
-
return [theme.fg("accent", theme.bold("Extensions"))];
|
|
228
|
+
return [theme.fg("accent", theme.bold("Extensions")), theme.fg("muted", CONTRIB_NOTE)];
|
|
270
229
|
}
|
|
271
230
|
invalidate() { }
|
|
272
231
|
})());
|
package/dist/loader/loader.d.ts
CHANGED
|
@@ -96,6 +96,4 @@ export declare function getModuleStates(_ctx?: ExtensionContext): Record<string,
|
|
|
96
96
|
export declare function resetModuleOverrides(_ctx?: ExtensionContext): void;
|
|
97
97
|
/** Get all valid config reference names for API completion/docs. */
|
|
98
98
|
export declare function getModuleRefs(): string[];
|
|
99
|
-
/** @deprecated Use getModuleRefs. Kept as an alias for existing callers. */
|
|
100
|
-
export declare function getAllRefNames(): string[];
|
|
101
99
|
//# sourceMappingURL=loader.d.ts.map
|
package/dist/loader/loader.js
CHANGED
|
@@ -571,8 +571,4 @@ function topologicalSort(extIds) {
|
|
|
571
571
|
export function getModuleRefs() {
|
|
572
572
|
return allConfigRefs();
|
|
573
573
|
}
|
|
574
|
-
/** @deprecated Use getModuleRefs. Kept as an alias for existing callers. */
|
|
575
|
-
export function getAllRefNames() {
|
|
576
|
-
return getModuleRefs();
|
|
577
|
-
}
|
|
578
574
|
//# sourceMappingURL=loader.js.map
|
package/docs/LOCAL_DEV.md
CHANGED
|
@@ -138,7 +138,7 @@ before and after a full run. A non-zero delta is the leak this rule prevents.
|
|
|
138
138
|
### New fixtures: use `node:test`
|
|
139
139
|
|
|
140
140
|
A new fixture is written on `node:test`. The runner does not care which style a
|
|
141
|
-
file uses -
|
|
141
|
+
file uses - 117 of the core suite's 353 fixtures already use `node:test` - so the
|
|
142
142
|
reason is the failure the other style produces. A hand-rolled helper that calls
|
|
143
143
|
`fn()` without awaiting it discards the case's promise, so an `async` case runs
|
|
144
144
|
detached and the file can print its green tally before that case settles, with
|
package/docs/SETUP.md
CHANGED
|
@@ -397,7 +397,7 @@ braid's loader follows a four-tier precedence (high → low):
|
|
|
397
397
|
| `/modules list` | Print every extension, its description and where its state came from |
|
|
398
398
|
| `/modules reset` | Clear in-process overrides (persisted `braid.modules` stays) |
|
|
399
399
|
|
|
400
|
-
The
|
|
400
|
+
The list covers the core manifest only. A contrib extension has no entry here: it is toggled through its `braid.contrib.<name>` key in `~/.pi/agent/settings.json`.
|
|
401
401
|
|
|
402
402
|
### `BRAID_MODULES` syntax
|
|
403
403
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.43.
|
|
3
|
+
"version": "0.43.42",
|
|
4
4
|
"description": "🥨 Braid - Metacognitive extension suite for pi coding agent: Adds memory, plan tracking, structured cognitive signals and completion checks on top of pi core loop, so long sessions keep their context and stop claiming unfinished work.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -202,6 +202,18 @@ After release:
|
|
|
202
202
|
- Commit stable package versions before remote npm publication. Local snapshots
|
|
203
203
|
use the derived iteration version instead.
|
|
204
204
|
- Use `npm version` for semver bumps in the target workspace.
|
|
205
|
+
- The bump is chosen by what a consumer can notice, not by how many patches
|
|
206
|
+
have accumulated:
|
|
207
|
+
|
|
208
|
+
| Bump | Trigger |
|
|
209
|
+
| ----- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
210
|
+
| patch | nothing a caller can observe: a fix, an internal refactor, docs, a test |
|
|
211
|
+
| minor | a caller must observe it: a new tool, command or flag; a new setting key; a changed default; a new required field in a contract; a deprecation |
|
|
212
|
+
| major | a break that needs a migration step: a renamed setting, a removed field, a changed wire or on-disk format |
|
|
213
|
+
|
|
214
|
+
A long patch line is not a reason to cut a minor on its own; it is a prompt to
|
|
215
|
+
ask whether one of the changes above already happened and was mislabelled.
|
|
216
|
+
|
|
205
217
|
- A bare core release tag (`v<core version>`) runs the publish workflow. Publish
|
|
206
218
|
a sibling package by dispatching from `main` with `only=<workspace directory>`
|
|
207
219
|
(`core|skills|providers|telegram|contrib`, not the npm package name).
|
|
@@ -30,8 +30,8 @@ and emits alerts on degradation/anomaly.
|
|
|
30
30
|
|
|
31
31
|
_(None - both `get_session_metrics` and `get_tool_reliability` removed in <!-- doc-ref-exempt: removed tool named as history -->
|
|
32
32
|
2026-06 tool-inventory audit. They were passive observability with no
|
|
33
|
-
internal consumers. Background tracking and the
|
|
34
|
-
|
|
33
|
+
internal consumers. Background tracking and the `/analytics all` scope
|
|
34
|
+
remain.)_
|
|
35
35
|
|
|
36
36
|
## Event-bus contract
|
|
37
37
|
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
} from "../../../lib/state/session-start-queue.js";
|
|
18
18
|
import { existsSync, mkdirSync } from "node:fs";
|
|
19
19
|
import { metricsDir, metricsFilePath } from "../../../lib/config/braid-paths.js";
|
|
20
|
-
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
|
+
import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
21
21
|
import { safeSessionManagerTyped } from "../../../lib/state/extension-state.js";
|
|
22
22
|
import { emitSignal } from "../../../lib/reasoning/belief-policy-engine.js";
|
|
23
23
|
import { eventBus } from "../../../lib/events/event-bus.js";
|
|
@@ -53,6 +53,40 @@ import {
|
|
|
53
53
|
|
|
54
54
|
const log = createLogger("analytics");
|
|
55
55
|
|
|
56
|
+
/** The three reports one command prints, keyed by the verb a script types. */
|
|
57
|
+
const ANALYTICS_SCOPES: Record<
|
|
58
|
+
string,
|
|
59
|
+
{ label: string; render: (ctx: ExtensionCommandContext) => string }
|
|
60
|
+
> = {
|
|
61
|
+
session: {
|
|
62
|
+
label: "This session",
|
|
63
|
+
render: (ctx) => formatCurrentSession(getAnalyticsStateInternal(ctx)),
|
|
64
|
+
},
|
|
65
|
+
all: { label: "All sessions", render: () => formatAllSessions() },
|
|
66
|
+
ab: { label: "A/B variants", render: () => formatABResults() },
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
/** Print one scope through the formatter that scope already owned. */
|
|
70
|
+
function showAnalyticsScope(scope: string, ctx: ExtensionCommandContext): void {
|
|
71
|
+
ctx.ui.notify(ANALYTICS_SCOPES[scope].render(ctx), "info");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** No argument: ask the scope in the TUI; headless prints the default scope. */
|
|
75
|
+
async function pickAnalyticsScope(ctx: ExtensionCommandContext): Promise<void> {
|
|
76
|
+
if (ctx.mode !== "tui") {
|
|
77
|
+
ctx.ui.notify(
|
|
78
|
+
`${ANALYTICS_SCOPES.session.render(ctx)}\n\nOther scopes: /analytics all, /analytics ab`,
|
|
79
|
+
"info",
|
|
80
|
+
);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const labels = Object.values(ANALYTICS_SCOPES).map((scope) => scope.label);
|
|
84
|
+
const chosen = await ctx.ui.select("Analytics scope", labels);
|
|
85
|
+
if (chosen === undefined) return;
|
|
86
|
+
const scope = Object.keys(ANALYTICS_SCOPES).find((key) => ANALYTICS_SCOPES[key].label === chosen);
|
|
87
|
+
if (scope) showAnalyticsScope(scope, ctx);
|
|
88
|
+
}
|
|
89
|
+
|
|
56
90
|
export default function analyticsExtension(pi: ExtensionAPI) {
|
|
57
91
|
registerSessionStartHandler(pi, SessionPhase.MemoryState, async (_event, ctx) => {
|
|
58
92
|
resetCriticStats();
|
|
@@ -344,30 +378,27 @@ export default function analyticsExtension(pi: ExtensionAPI) {
|
|
|
344
378
|
});
|
|
345
379
|
|
|
346
380
|
pi.registerCommand("analytics", {
|
|
347
|
-
description:
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
description: "Show A/B experiment results across variants",
|
|
362
|
-
handler: async (_args, ctx) => {
|
|
363
|
-
ctx.ui.notify(formatABResults(), "info");
|
|
381
|
+
description:
|
|
382
|
+
"Show tool analytics. No argument asks the scope in the TUI. " +
|
|
383
|
+
"Usage: /analytics [session|all|ab]",
|
|
384
|
+
handler: async (args, ctx) => {
|
|
385
|
+
const scope = (args ?? "").trim().toLowerCase();
|
|
386
|
+
if (scope === "") {
|
|
387
|
+
await pickAnalyticsScope(ctx);
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
if (!(scope in ANALYTICS_SCOPES)) {
|
|
391
|
+
ctx.ui.notify(`Unknown scope "${scope}". Usage: /analytics [session|all|ab]`, "warning");
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
showAnalyticsScope(scope, ctx);
|
|
364
395
|
},
|
|
365
396
|
});
|
|
366
397
|
|
|
367
398
|
// No agent-callable observability tools here (session metrics / tool
|
|
368
399
|
// reliability): the agent rarely needs self-observability and no internal
|
|
369
400
|
// caller used them. Background tracking (toolMetrics, sessionStartTime)
|
|
370
|
-
// still runs and feeds event-bus alerts and the analytics
|
|
401
|
+
// still runs and feeds event-bus alerts and the analytics scopes; the
|
|
371
402
|
// A/B variant is set programmatically via
|
|
372
403
|
// getAnalyticsStateInternal(ctx).abVariant.
|
|
373
404
|
}
|
|
@@ -100,10 +100,9 @@ Whispers from compaction-retry, navigation, and memory hooks use `triggerTurn: t
|
|
|
100
100
|
|
|
101
101
|
- **compaction-retry** (`context-manager/handlers/compaction-retry.ts`): exhausted / context-window / threshold errors all wake the agent. In-progress retry messages (`overflow`, `transient`, `threshold`, `summary-cap`) park (`delivery: "park"`, no forced continuation turn) because the agent is mid-retry; a summary-cap failure self-recovers on the retry cycle with a bounded chunked summary (compaction-chunked.ts), so only its exhausted path wakes.
|
|
102
102
|
- **navigation** (`context-manager/handlers/navigation.ts`): "Context checkout prepared" with a direct action instruction wakes the agent.
|
|
103
|
-
- **three-memory-stores** (`memory/three-memory-stores/index.ts`): tool reliability alerts (gap > 0.2) wake the agent so it can switch tools.
|
|
104
103
|
- **rotation** (`context-manager/handlers/shared.ts:327`, `rotation.ts`): emergency rotation wakes. The size notice parks while a run is streaming (flushed at the same `turn_end`) and keeps its wake only when the session is idle; a notice still in flight suppresses a second emission through the unresolved-advisory registry in `lib/utils/injection-tracker.ts`.
|
|
105
104
|
|
|
106
|
-
Advisory whispers (memory recall, past insights, hint injection, SOP suggestions) park (`delivery: "park"`) - they provide context for the agent's natural flow.
|
|
105
|
+
Advisory whispers (memory recall, past insights, hint injection, SOP suggestions, and the tool-reliability hint from `three-memory-stores`) park (`delivery: "park"`) - they provide context for the agent's natural flow. The reliability hint is scoped hard because it names a tool: the alternative must belong to the failing tool's declared family (`lib/tools/tool-families.ts`), both sides need at least 5 attempts, the gap must reach 0.2, the sentence prints both samples, and it fires at most once per family per `HINT_COOLDOWN_TURNS` turns.
|
|
107
106
|
|
|
108
107
|
## Re-entrancy safety
|
|
109
108
|
|
|
@@ -57,7 +57,14 @@ import type { MemoryStoresState, Episode } from "./types.js";
|
|
|
57
57
|
// Sub-module imports
|
|
58
58
|
import { retrieveByRUMS, sanitizeEpisode } from "./episodes.js";
|
|
59
59
|
import { addHeuristicRule, getPrioritizedHeuristics, BUILTIN_HEURISTICS } from "./heuristics.js";
|
|
60
|
-
import { recordToolCall
|
|
60
|
+
import { recordToolCall } from "./tool-reliability.js";
|
|
61
|
+
import {
|
|
62
|
+
allowFamilyHint,
|
|
63
|
+
createFamilyHintGate,
|
|
64
|
+
formatSuggestion,
|
|
65
|
+
noteTurn,
|
|
66
|
+
suggestAlternative,
|
|
67
|
+
} from "../../../lib/tools/tool-families.js";
|
|
61
68
|
import {
|
|
62
69
|
whisperRecall,
|
|
63
70
|
AUTO_RECALL_ENABLED,
|
|
@@ -246,47 +253,39 @@ export default function threeMemoryStoresExtension(pi: ExtensionAPI) {
|
|
|
246
253
|
);
|
|
247
254
|
}
|
|
248
255
|
|
|
256
|
+
// bash is never the tool to abandon: it runs anything, so a swipe
|
|
257
|
+
// against it is not advice a reader can act on.
|
|
249
258
|
if (toolName !== "bash") {
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
// means the current tool is consistently failing for this
|
|
281
|
-
// task type. Wakes the agent so it can switch tools.
|
|
282
|
-
{ deliverAs: "steer", triggerTurn: true },
|
|
283
|
-
// type "heartbeat" (not "other") keeps the _braidInjected /
|
|
284
|
-
// _braidSource decoration; heartbeat is the analytics-alert
|
|
285
|
-
// type (see heartbeat/analytics). Source root "analytics" so
|
|
286
|
-
// envelope family == body family ().
|
|
287
|
-
{ type: "heartbeat", source: "analytics/reliability" },
|
|
288
|
-
);
|
|
289
|
-
}
|
|
259
|
+
// `taskType` groups the statistics; only a declared substitute may be
|
|
260
|
+
// named (lib/tools/tool-families.ts). The suggestion carries its own
|
|
261
|
+
// sample, so the body prints the counts the claim rests on and thin
|
|
262
|
+
// data cannot reach a reader.
|
|
263
|
+
const suggestion = suggestAlternative({
|
|
264
|
+
stats: state.toolStats.values(),
|
|
265
|
+
tool: toolName,
|
|
266
|
+
taskType,
|
|
267
|
+
});
|
|
268
|
+
const hintGate = getExtensionState(ctx, "tool-family-hints", createFamilyHintGate);
|
|
269
|
+
if (suggestion && allowFamilyHint(hintGate, suggestion.family)) {
|
|
270
|
+
sendTrackedMessage(
|
|
271
|
+
pi,
|
|
272
|
+
{
|
|
273
|
+
// This body is TOOL-RELIABILITY ANALYTICS
|
|
274
|
+
// (a stats-driven recommendation), not memory recall. True
|
|
275
|
+
// family is analytics; the envelope derives from the source
|
|
276
|
+
// below, so both say analytics.
|
|
277
|
+
customType: "analytics",
|
|
278
|
+
display: true,
|
|
279
|
+
content: [{ type: "text", text: formatSuggestion(suggestion) }],
|
|
280
|
+
},
|
|
281
|
+
// An advisory hint rides the next turn; it does not create one.
|
|
282
|
+
{ delivery: "park", deliverAs: "steer" },
|
|
283
|
+
// type "heartbeat" (not "other") keeps the _braidInjected /
|
|
284
|
+
// _braidSource decoration; heartbeat is the analytics-alert
|
|
285
|
+
// type (see heartbeat/analytics). Source root "analytics" so
|
|
286
|
+
// envelope family == body family.
|
|
287
|
+
{ type: "heartbeat", source: "analytics/reliability" },
|
|
288
|
+
);
|
|
290
289
|
}
|
|
291
290
|
}
|
|
292
291
|
}
|
|
@@ -299,6 +298,9 @@ export default function threeMemoryStoresExtension(pi: ExtensionAPI) {
|
|
|
299
298
|
pi.on(TURN_END, async (_event: unknown, ctx: ExtensionContext) => {
|
|
300
299
|
try {
|
|
301
300
|
const state = getState(ctx);
|
|
301
|
+
// The family-rate-limit clock runs on completed turns, so it advances
|
|
302
|
+
// before this handler's own early returns can skip it.
|
|
303
|
+
noteTurn(getExtensionState(ctx, "tool-family-hints", createFamilyHintGate));
|
|
302
304
|
if (!AUTO_RECALL_ENABLED || state.currentSessionTools.length < 3) return;
|
|
303
305
|
|
|
304
306
|
const now = Date.now();
|
package/src/index.ts
CHANGED
|
@@ -18,8 +18,6 @@
|
|
|
18
18
|
* Subagent API:
|
|
19
19
|
* Researcher subagents can call `/modules` (verbs: on, off, toggle, reset,
|
|
20
20
|
* all on|off, list) to control extension loading during benchmark experiments.
|
|
21
|
-
* The six former names (modules-enable, modules-disable, modules-toggle,
|
|
22
|
-
* modules-reset, modules-list, modules-status) forward to it for one release.
|
|
23
21
|
*/
|
|
24
22
|
|
|
25
23
|
import { readFileSync } from "node:fs";
|
package/src/lib/AGENTS.md
CHANGED
|
@@ -29,6 +29,7 @@ shared conventions.
|
|
|
29
29
|
| [`harness/`](./harness/AGENTS.md) | Self-harness loop - failure-trace collection, weakness mining, candidate proposal and validation. |
|
|
30
30
|
| `orchestrator/` | Multi-agent coordination - file-lock registry, worker registry with close-time gate-row well-formedness, the production-write preflight read, and the population-stamp read the merge path compares against the repo, worker branches, conflict detection, merge verification. |
|
|
31
31
|
| [`telemetry/`](./telemetry/AGENTS.md) | Bounded telemetry accumulators for the degradation-detection loop (drift signals, context/RSS heartbeat). |
|
|
32
|
+
| `tools/` | Tool catalog - which tools may be compared on reliability, and the substitution rule built on it. |
|
|
32
33
|
| `verification/` | Cross-file verification - downstream import tracing and derived-file coordination checks. |
|
|
33
34
|
|
|
34
35
|
## Conventions
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool families - which tools may be compared on reliability.
|
|
3
|
+
*
|
|
4
|
+
* A reliability statistic answers "how often did this tool work?" only inside a
|
|
5
|
+
* population of attempts that could have been made by either tool. `taskType`
|
|
6
|
+
* (see `inferTaskType` in the memory extension) is not that population: it names
|
|
7
|
+
* a phase of work ("general", "file_edit", "search"), and a tool with no entry in
|
|
8
|
+
* its map falls through to "general". That label therefore groups
|
|
9
|
+
* `herdr_layout` (arranges terminal panes) with `fetch_content` (fetches a URL),
|
|
10
|
+
* and a comparison built on it reads as advice to fetch a web page with the
|
|
11
|
+
* layout tool.
|
|
12
|
+
*
|
|
13
|
+
* **`taskType` is a label, not a claim about what a tool does.** The claim lives
|
|
14
|
+
* here: a tool is substitutable only with tools in its own family. A tool with no
|
|
15
|
+
* entry in this map has no family, is not known to be comparable with anything,
|
|
16
|
+
* and never produces a recommendation - a new tool joins the family map by being
|
|
17
|
+
* declared, not by being observed.
|
|
18
|
+
*
|
|
19
|
+
* The module is pure: no state, no I/O, no imports. Callers own the statistics and
|
|
20
|
+
* the session-scoped hint gate.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export type ToolFamily = "web-fetch" | "terminal" | "files" | "agents" | "memory" | "orchestration";
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Which tools do the same job. Membership is a declaration about purpose, not a
|
|
27
|
+
* statement that the tools are interchangeable in every argument shape: `read`
|
|
28
|
+
* and `edit` are both files tools, and one reads what the other writes.
|
|
29
|
+
*/
|
|
30
|
+
export const TOOL_FAMILIES: Readonly<Record<string, ToolFamily>> = {
|
|
31
|
+
fetch_content: "web-fetch",
|
|
32
|
+
web_search: "web-fetch",
|
|
33
|
+
get_search_content: "web-fetch",
|
|
34
|
+
|
|
35
|
+
bash: "terminal",
|
|
36
|
+
herdr_pane: "terminal",
|
|
37
|
+
herdr_layout: "terminal",
|
|
38
|
+
|
|
39
|
+
read: "files",
|
|
40
|
+
write: "files",
|
|
41
|
+
edit: "files",
|
|
42
|
+
grep: "files",
|
|
43
|
+
find: "files",
|
|
44
|
+
ls: "files",
|
|
45
|
+
|
|
46
|
+
herdr_agent: "agents",
|
|
47
|
+
|
|
48
|
+
memory: "memory",
|
|
49
|
+
memory_store: "memory",
|
|
50
|
+
scratchpad: "memory",
|
|
51
|
+
session: "memory",
|
|
52
|
+
|
|
53
|
+
orchestrate: "orchestration",
|
|
54
|
+
timer: "orchestration",
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The family of `tool`, or `null` when the tool is not declared. The own-property
|
|
59
|
+
* test is deliberate: an index lookup for a name like "toString" would otherwise
|
|
60
|
+
* return something off the prototype chain and read as a family.
|
|
61
|
+
*/
|
|
62
|
+
export function familyOf(tool: string): ToolFamily | null {
|
|
63
|
+
return Object.prototype.hasOwnProperty.call(TOOL_FAMILIES, tool) ? TOOL_FAMILIES[tool] : null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The part of a tool statistic this module reads. */
|
|
67
|
+
export interface ToolStatLike {
|
|
68
|
+
tool: string;
|
|
69
|
+
taskType: string;
|
|
70
|
+
attempts: number;
|
|
71
|
+
successes: number;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface Sample {
|
|
75
|
+
attempts: number;
|
|
76
|
+
successes: number;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface ToolFamilySuggestion {
|
|
80
|
+
/** The tool to switch to. Same family as `tool`. */
|
|
81
|
+
alternative: string;
|
|
82
|
+
/** Reliability of `tool` for `taskType`, in [0, 1]. */
|
|
83
|
+
current: number;
|
|
84
|
+
/** Reliability of `alternative` for the same task type, in [0, 1]. */
|
|
85
|
+
better: number;
|
|
86
|
+
/** The failing tool's own sample. */
|
|
87
|
+
sample: Sample;
|
|
88
|
+
/** The alternative's sample. */
|
|
89
|
+
alternatives: Sample;
|
|
90
|
+
/** The tool whose failure prompted this. */
|
|
91
|
+
tool: string;
|
|
92
|
+
family: ToolFamily;
|
|
93
|
+
taskType: string;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Both sides must carry at least this many attempts for the task type. */
|
|
97
|
+
export const MIN_ATTEMPTS = 5;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The alternative must beat the failing tool by this margin. Held as the fraction
|
|
101
|
+
* 1/5 so the decision runs on integer counts (see `meetsGap`) instead of the
|
|
102
|
+
* subtraction of two rounded ratios.
|
|
103
|
+
*/
|
|
104
|
+
export const MIN_GAP = 0.2;
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* `alternative` beats `current` by at least `MIN_GAP`, decided exactly:
|
|
108
|
+
*
|
|
109
|
+
* alt.s/alt.a - cur.s/cur.a >= 1/5 <=> 5*(alt.s*cur.a - cur.s*alt.a) >= cur.a*alt.a
|
|
110
|
+
*
|
|
111
|
+
* The integer form matters because float subtraction puts a gap of exactly 0.2
|
|
112
|
+
* (3/5 vs 2/5) at 0.19999999999999996, which would refuse a comparison the stated
|
|
113
|
+
* rule allows.
|
|
114
|
+
*/
|
|
115
|
+
function meetsGap(current: ToolStatLike, alternative: ToolStatLike): boolean {
|
|
116
|
+
const numerator =
|
|
117
|
+
alternative.successes * current.attempts - current.successes * alternative.attempts;
|
|
118
|
+
return numerator > 0 && 5 * numerator >= current.attempts * alternative.attempts;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The best substitute for `tool` on `taskType`, or `null` when there is none.
|
|
123
|
+
*
|
|
124
|
+
* Null whenever: the tool has no declared family; the tool's own sample for this
|
|
125
|
+
* task type is under `MIN_ATTEMPTS`; no same-family tool reaches `MIN_ATTEMPTS`
|
|
126
|
+
* on that task type; or no such tool clears `MIN_GAP`. Ties in reliability go to
|
|
127
|
+
* the lexicographically first tool name, so the same statistics always name the
|
|
128
|
+
* same alternative.
|
|
129
|
+
*/
|
|
130
|
+
export function suggestAlternative(input: {
|
|
131
|
+
stats: Iterable<ToolStatLike>;
|
|
132
|
+
tool: string;
|
|
133
|
+
taskType: string;
|
|
134
|
+
}): ToolFamilySuggestion | null {
|
|
135
|
+
const family = familyOf(input.tool);
|
|
136
|
+
if (family === null) return null;
|
|
137
|
+
|
|
138
|
+
let current: ToolStatLike | null = null;
|
|
139
|
+
const candidates: ToolStatLike[] = [];
|
|
140
|
+
for (const stat of input.stats) {
|
|
141
|
+
if (!stat || stat.taskType !== input.taskType) continue;
|
|
142
|
+
if (stat.tool === input.tool) {
|
|
143
|
+
if (current === null) current = stat;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (familyOf(stat.tool) === family) candidates.push(stat);
|
|
147
|
+
}
|
|
148
|
+
if (current === null || current.attempts < MIN_ATTEMPTS) return null;
|
|
149
|
+
const failing = current;
|
|
150
|
+
|
|
151
|
+
const qualified = candidates.filter(
|
|
152
|
+
(candidate) => candidate.attempts >= MIN_ATTEMPTS && meetsGap(failing, candidate),
|
|
153
|
+
);
|
|
154
|
+
if (qualified.length === 0) return null;
|
|
155
|
+
|
|
156
|
+
qualified.sort((a, b) => {
|
|
157
|
+
// Descending reliability by cross-multiplication; then by name for stability.
|
|
158
|
+
const byRate = b.successes * a.attempts - a.successes * b.attempts;
|
|
159
|
+
if (byRate !== 0) return byRate;
|
|
160
|
+
return a.tool < b.tool ? -1 : a.tool > b.tool ? 1 : 0;
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
const best = qualified[0];
|
|
164
|
+
return {
|
|
165
|
+
alternative: best.tool,
|
|
166
|
+
current: failing.successes / failing.attempts,
|
|
167
|
+
better: best.successes / best.attempts,
|
|
168
|
+
sample: { attempts: failing.attempts, successes: failing.successes },
|
|
169
|
+
alternatives: { attempts: best.attempts, successes: best.successes },
|
|
170
|
+
tool: input.tool,
|
|
171
|
+
family,
|
|
172
|
+
taskType: input.taskType,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The sentence a reader receives. Both samples are on the line: a percentage
|
|
178
|
+
* without its counts is what made the old hint unreadable, and a reader must be
|
|
179
|
+
* able to watch the sample decide the claim. `taskType` is named because it is
|
|
180
|
+
* what the counts are grouped by; the family is named because it is what makes
|
|
181
|
+
* the comparison admissible in the first place.
|
|
182
|
+
*/
|
|
183
|
+
export function formatSuggestion(suggestion: ToolFamilySuggestion): string {
|
|
184
|
+
const better = suggestion.alternatives;
|
|
185
|
+
const current = suggestion.sample;
|
|
186
|
+
return (
|
|
187
|
+
`${suggestion.alternative} succeeded ${better.successes} of ${better.attempts} on ` +
|
|
188
|
+
`"${suggestion.taskType}" where ${suggestion.tool} succeeded ${current.successes} of ` +
|
|
189
|
+
`${current.attempts} (same tool family: ${suggestion.family})`
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* One hint per family per this many turns.
|
|
195
|
+
*
|
|
196
|
+
* The statistics are cumulative for the session, so between two turns the sample
|
|
197
|
+
* grows while the direction of the claim does not change: a second copy of the
|
|
198
|
+
* same advice carries no fact the first lacked, and a reader who sees it every
|
|
199
|
+
* turn learns to filter the family out. Ten turns is long enough for a reader to
|
|
200
|
+
* act (switching tools shows up as new attempts on both sides) and short enough
|
|
201
|
+
* that a session which ignored the hint still sees it once more before it ends.
|
|
202
|
+
*/
|
|
203
|
+
export const HINT_COOLDOWN_TURNS = 10;
|
|
204
|
+
|
|
205
|
+
/** Per-session hint limiter. Hold it in session state, never at module scope. */
|
|
206
|
+
export interface FamilyHintGate {
|
|
207
|
+
/** Turns completed in this session. */
|
|
208
|
+
turn: number;
|
|
209
|
+
/** Turn of the last emitted hint, per family. */
|
|
210
|
+
lastHintTurn: Partial<Record<ToolFamily, number>>;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function createFamilyHintGate(): FamilyHintGate {
|
|
214
|
+
return { turn: 0, lastHintTurn: {} };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Advance the session's turn clock by one completed turn. */
|
|
218
|
+
export function noteTurn(gate: FamilyHintGate): void {
|
|
219
|
+
gate.turn += 1;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Claim the right to emit one hint for `family`. Returns `false` while the family
|
|
224
|
+
* is still inside its cooldown; a `true` return records the hint at the current
|
|
225
|
+
* turn, so the caller must emit when it gets one.
|
|
226
|
+
*/
|
|
227
|
+
export function allowFamilyHint(gate: FamilyHintGate, family: ToolFamily): boolean {
|
|
228
|
+
const last = gate.lastHintTurn[family];
|
|
229
|
+
if (last !== undefined && gate.turn - last < HINT_COOLDOWN_TURNS) return false;
|
|
230
|
+
gate.lastHintTurn[family] = gate.turn;
|
|
231
|
+
return true;
|
|
232
|
+
}
|