vigiles 12.1.0 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -10
- package/action.yml +73 -0
- package/dist/audit-report.d.ts +11 -0
- package/dist/audit-report.js +1 -0
- package/dist/audit-report.template.html +36 -26
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +0 -1
- package/dist/cli.js +62 -135
- package/dist/eval.js +17 -0
- package/dist/observe.d.ts +109 -0
- package/dist/observe.js +164 -0
- package/dist/scaffold-test.js +3 -2
- package/package.json +1 -1
- package/skills/debug-my-harness/SKILL.md +56 -0
- package/dist/core/hook-spec.d.ts +0 -74
- package/dist/core/hook-spec.js +0 -130
package/dist/cli-commands.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* recognizes exactly these, so this list can't silently drift from the code.
|
|
12
12
|
*/
|
|
13
13
|
/** Human-facing verbs (printed in help; typed by a human/agent/CI). */
|
|
14
|
-
export declare const VERBS: readonly ["init", "compile", "eject", "lint", "test", "eval", "audit", "
|
|
14
|
+
export declare const VERBS: readonly ["init", "compile", "eject", "lint", "test", "eval", "audit", "generate", "hook-runtime"];
|
|
15
15
|
/** Runtime entrypoint kinds under `vigiles hook-runtime <kind>` (emitted, not typed). */
|
|
16
16
|
export declare const HOOK_RUNTIME_KINDS: readonly ["run-program", "agent", "agent-start", "agent-done", "skill", "skill-tool", "skill-start", "skill-done", "run-skill", "intercept-tool", "guard", "action", "refs", "eval-lock-nudge", "effect-enter", "effect-exit"];
|
|
17
17
|
export type Verb = (typeof VERBS)[number];
|
package/dist/cli-commands.js
CHANGED
package/dist/cli.js
CHANGED
|
@@ -22,8 +22,6 @@ const cli_flags_js_1 = require("./cli-flags.js");
|
|
|
22
22
|
const setup_plan_js_1 = require("./setup-plan.js");
|
|
23
23
|
const types_js_1 = require("./core/types.js");
|
|
24
24
|
const test_coverage_js_1 = require("./test-coverage.js");
|
|
25
|
-
const scaffold_test_js_1 = require("./scaffold-test.js");
|
|
26
|
-
const effects_js_1 = require("./core/effects.js");
|
|
27
25
|
const scan_js_1 = require("./scan.js");
|
|
28
26
|
const scan_trigger_suggest_js_1 = require("./scan-trigger-suggest.js");
|
|
29
27
|
const dialect_drift_js_1 = require("./dialect-drift.js");
|
|
@@ -52,6 +50,7 @@ const hook_install_js_1 = require("./hook-install.js");
|
|
|
52
50
|
const hook_providers_js_1 = require("./core/hook-providers.js");
|
|
53
51
|
const toml_1 = require("@iarna/toml");
|
|
54
52
|
const agent_runtime_js_1 = require("./adapters/claude-code/agent-runtime.js");
|
|
53
|
+
const observe_js_1 = require("./observe.js");
|
|
55
54
|
const effect_region_js_1 = require("./adapters/claude-code/effect-region.js");
|
|
56
55
|
const tool_intercept_js_1 = require("./tool-intercept.js");
|
|
57
56
|
const refs_js_1 = require("./core/refs.js");
|
|
@@ -3611,135 +3610,6 @@ function capabilitiesOfReport(report, dialect) {
|
|
|
3611
3610
|
}));
|
|
3612
3611
|
return (0, generate_harness_js_1.computeHarnessCapabilities)(agents, dialect);
|
|
3613
3612
|
}
|
|
3614
|
-
/**
|
|
3615
|
-
* The plugin's declared name for the namespaced skill id, read from the layout's
|
|
3616
|
-
* manifest (adapter-aware path, not a hardcoded `.claude-plugin/`), falling back to
|
|
3617
|
-
* the dir basename. JSON manifests only for now (a TOML/Codex manifest → basename).
|
|
3618
|
-
*/
|
|
3619
|
-
function pluginNameFor(dir, manifestPath) {
|
|
3620
|
-
try {
|
|
3621
|
-
const manifest = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.resolve)(dir, manifestPath), "utf-8"));
|
|
3622
|
-
if (typeof manifest.name === "string" && manifest.name)
|
|
3623
|
-
return manifest.name;
|
|
3624
|
-
}
|
|
3625
|
-
catch {
|
|
3626
|
-
/* missing / non-JSON manifest → fall back */
|
|
3627
|
-
}
|
|
3628
|
-
return (0, node_path_1.basename)(dir);
|
|
3629
|
-
}
|
|
3630
|
-
/** Enrich an untested Surface with the metadata the right template needs. */
|
|
3631
|
-
/** Extract `"name": type` fields from one rendered `vigiles:ok`/`err` shape block. */
|
|
3632
|
-
function parseContractFields(block) {
|
|
3633
|
-
const fields = [];
|
|
3634
|
-
const re = /"([^"]+)"\s*:\s*(string\[\]|string|number|boolean)/g;
|
|
3635
|
-
let m;
|
|
3636
|
-
while ((m = re.exec(block)) !== null) {
|
|
3637
|
-
fields.push({ name: m[1], type: m[2] });
|
|
3638
|
-
}
|
|
3639
|
-
return fields;
|
|
3640
|
-
}
|
|
3641
|
-
/**
|
|
3642
|
-
* Parse a subagent's compiled `## Output contract` (the `vigiles:ok` / `vigiles:err`
|
|
3643
|
-
* blocks the compiler emits) back into a typed `ResultContract`, so the generator
|
|
3644
|
-
* can write an `assertAgentOk` test against the real fields. Returns null when the
|
|
3645
|
-
* agent has no result() contract.
|
|
3646
|
-
*/
|
|
3647
|
-
function parseResultContract(md) {
|
|
3648
|
-
const ok = /```vigiles:ok\n([\s\S]*?)```/.exec(md);
|
|
3649
|
-
const err = /```vigiles:err\n([\s\S]*?)```/.exec(md);
|
|
3650
|
-
if (!ok && !err)
|
|
3651
|
-
return null;
|
|
3652
|
-
const okFields = ok ? parseContractFields(ok[1]) : [];
|
|
3653
|
-
const errFields = err ? parseContractFields(err[1]) : [];
|
|
3654
|
-
if (okFields.length === 0 && errFields.length === 0)
|
|
3655
|
-
return null;
|
|
3656
|
-
return { ok: okFields, err: errFields };
|
|
3657
|
-
}
|
|
3658
|
-
function scaffoldInputFor(s, report, pluginName, dir, dialect) {
|
|
3659
|
-
const base = { kind: s.kind, name: s.name, path: s.path };
|
|
3660
|
-
switch (s.kind) {
|
|
3661
|
-
case "skill": {
|
|
3662
|
-
const sk = report.skills.find((x) => x.name === s.name);
|
|
3663
|
-
return { ...base, pluginName, userInvoked: sk?.userInvoked };
|
|
3664
|
-
}
|
|
3665
|
-
case "agent": {
|
|
3666
|
-
const ag = report.agents.find((x) => x.name === s.name);
|
|
3667
|
-
const tools = ag?.tools ?? null;
|
|
3668
|
-
const sideEffectingTools = tools
|
|
3669
|
-
? (0, effects_js_1.effectSurface)(tools, dialect).sideEffecting
|
|
3670
|
-
: undefined;
|
|
3671
|
-
let resultContract = null;
|
|
3672
|
-
try {
|
|
3673
|
-
resultContract = parseResultContract((0, node_fs_1.readFileSync)((0, node_path_1.resolve)(dir, s.path), "utf-8"));
|
|
3674
|
-
}
|
|
3675
|
-
catch {
|
|
3676
|
-
// agent .md unreadable → no contract to generate against
|
|
3677
|
-
}
|
|
3678
|
-
return { ...base, tools, sideEffectingTools, resultContract };
|
|
3679
|
-
}
|
|
3680
|
-
case "hook":
|
|
3681
|
-
return { ...base, hookCommand: `bash ${s.path}` };
|
|
3682
|
-
}
|
|
3683
|
-
}
|
|
3684
|
-
/**
|
|
3685
|
-
* `vigiles scaffold-test [dir]` — generate a runnable STARTER test for each
|
|
3686
|
-
* untested skill/agent/hook (B1, test-gen from free-form). Reuses the
|
|
3687
|
-
* untested-surface detector for the list + `scan` for the metadata, then emits the
|
|
3688
|
-
* cheapest meaningful tier per kind (hook → `runHook`, skill → `measureTriggerRate`,
|
|
3689
|
-
* subagent → `runHarnessTest`) at the surface's suggested test path. Dry-run by
|
|
3690
|
-
* default (prints the scaffolds); `--write` creates the files (never clobbering an
|
|
3691
|
-
* existing one); `--json` for the agent-consumable `{ path, content }[]`.
|
|
3692
|
-
*/
|
|
3693
|
-
function handleScaffoldTest(restArgs, args) {
|
|
3694
|
-
const dir = (0, node_path_1.resolve)(restArgs[0] ?? ".");
|
|
3695
|
-
const write = args.includes("--write");
|
|
3696
|
-
const json = args.includes("--json");
|
|
3697
|
-
const harnessFlag = harnessFlagFrom(args);
|
|
3698
|
-
const adapter = harnessFlag
|
|
3699
|
-
? (0, adapter_registry_js_1.resolveAdapter)(dir, harnessFlag)
|
|
3700
|
-
: (0, adapter_registry_js_1.detectAdapterResult)(dir).adapter;
|
|
3701
|
-
const { untested } = (0, test_coverage_js_1.findUntestedSurfaces)({
|
|
3702
|
-
basePath: dir,
|
|
3703
|
-
layout: adapter.layout,
|
|
3704
|
-
});
|
|
3705
|
-
const report = (0, scan_js_1.scanPlugin)(dir, adapter.layout, adapter.dialect);
|
|
3706
|
-
const pluginName = pluginNameFor(dir, adapter.layout.manifestPath);
|
|
3707
|
-
const scaffolds = untested.map((s) => (0, scaffold_test_js_1.scaffoldTest)(scaffoldInputFor(s, report, pluginName, dir, adapter.dialect)));
|
|
3708
|
-
if (json) {
|
|
3709
|
-
console.log(JSON.stringify(scaffolds, null, 2));
|
|
3710
|
-
return;
|
|
3711
|
-
}
|
|
3712
|
-
if (!write) {
|
|
3713
|
-
console.log((0, scaffold_test_js_1.formatScaffolds)(scaffolds));
|
|
3714
|
-
for (const s of scaffolds) {
|
|
3715
|
-
console.log(`\n# ${s.path}\n`);
|
|
3716
|
-
console.log(s.content);
|
|
3717
|
-
}
|
|
3718
|
-
if (scaffolds.length > 0) {
|
|
3719
|
-
console.log("Re-run with --write to create these files.");
|
|
3720
|
-
}
|
|
3721
|
-
return;
|
|
3722
|
-
}
|
|
3723
|
-
const written = [];
|
|
3724
|
-
const skipped = [];
|
|
3725
|
-
for (const s of scaffolds) {
|
|
3726
|
-
const target = (0, node_path_1.resolve)(dir, s.path);
|
|
3727
|
-
if ((0, node_fs_1.existsSync)(target)) {
|
|
3728
|
-
skipped.push(s.path);
|
|
3729
|
-
continue;
|
|
3730
|
-
}
|
|
3731
|
-
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(target), { recursive: true });
|
|
3732
|
-
(0, node_fs_1.writeFileSync)(target, s.content);
|
|
3733
|
-
written.push(s.path);
|
|
3734
|
-
}
|
|
3735
|
-
for (const p of written)
|
|
3736
|
-
console.log(`✓ wrote ${p}`);
|
|
3737
|
-
for (const p of skipped)
|
|
3738
|
-
console.log(`⊘ skipped ${p} (already exists)`);
|
|
3739
|
-
if (written.length === 0 && skipped.length === 0) {
|
|
3740
|
-
console.log("Nothing to scaffold — every surface already has a test.");
|
|
3741
|
-
}
|
|
3742
|
-
}
|
|
3743
3613
|
function printUsage(command) {
|
|
3744
3614
|
console.log("vigiles — compile typed specs to instruction files");
|
|
3745
3615
|
console.log("");
|
|
@@ -3756,7 +3626,6 @@ function printUsage(command) {
|
|
|
3756
3626
|
console.log(" vigiles eval [files...] Run *.eval.mjs real-model harness evals (--trials=N, --min=N, --no-skip)");
|
|
3757
3627
|
console.log(" --update records each named eval's result to a committed lock (run locally on your subscription)");
|
|
3758
3628
|
console.log(" --check verifies committed eval results against current inputs WITHOUT a model — the CI staleness gate");
|
|
3759
|
-
console.log(" vigiles scaffold-test [dir] Generate a starter test for each untested skill/agent/hook (--write, --json)");
|
|
3760
3629
|
console.log("");
|
|
3761
3630
|
console.log("Examples:");
|
|
3762
3631
|
console.log(" vigiles init Auto-detect project, create specs, wire CI");
|
|
@@ -3862,6 +3731,11 @@ function skillStartCommand(target) {
|
|
|
3862
3731
|
process.exit(2);
|
|
3863
3732
|
}
|
|
3864
3733
|
(0, skill_runtime_js_1.setActiveSkill)(process.cwd(), target);
|
|
3734
|
+
// Record the fire in the flight recorder: the skill NAME is the parent dir of
|
|
3735
|
+
// its SKILL.md (skills/<name>/SKILL.md), falling back to the raw target.
|
|
3736
|
+
const parts = target.replace(/\\/g, "/").split("/").filter(Boolean);
|
|
3737
|
+
const name = parts.length >= 2 ? parts[parts.length - 2] : (parts[0] ?? target);
|
|
3738
|
+
(0, observe_js_1.appendObservation)({ kind: "skill", name, fired: true });
|
|
3865
3739
|
console.log(`Active skill: ${target}`);
|
|
3866
3740
|
}
|
|
3867
3741
|
/**
|
|
@@ -3962,6 +3836,13 @@ function agentHookCommand() {
|
|
|
3962
3836
|
return;
|
|
3963
3837
|
const decision = (0, agent_runtime_js_1.evaluatePreToolUse)(cwd, tool, command);
|
|
3964
3838
|
if (!decision.allow) {
|
|
3839
|
+
(0, observe_js_1.appendObservation)({
|
|
3840
|
+
kind: "agent",
|
|
3841
|
+
name: (0, agent_runtime_js_1.readActiveAgent)(cwd) ?? "unknown",
|
|
3842
|
+
tool,
|
|
3843
|
+
allowed: false,
|
|
3844
|
+
reason: decision.message,
|
|
3845
|
+
});
|
|
3965
3846
|
console.error(decision.message);
|
|
3966
3847
|
process.exit(2);
|
|
3967
3848
|
}
|
|
@@ -4510,10 +4391,26 @@ function emitGate(decision, on, mode, file) {
|
|
|
4510
4391
|
const action = (0, hook_program_js_1.gateAction)(decision, mode);
|
|
4511
4392
|
switch (action.kind) {
|
|
4512
4393
|
case "block":
|
|
4394
|
+
(0, observe_js_1.appendObservation)({
|
|
4395
|
+
kind: "hook",
|
|
4396
|
+
event: on,
|
|
4397
|
+
decision: "deny",
|
|
4398
|
+
mode: "enforce",
|
|
4399
|
+
rule: file,
|
|
4400
|
+
reason: action.reason,
|
|
4401
|
+
});
|
|
4513
4402
|
console.error(action.reason);
|
|
4514
4403
|
process.exit(2);
|
|
4515
4404
|
return;
|
|
4516
4405
|
case "ask":
|
|
4406
|
+
(0, observe_js_1.appendObservation)({
|
|
4407
|
+
kind: "hook",
|
|
4408
|
+
event: on,
|
|
4409
|
+
decision: "ask",
|
|
4410
|
+
mode: "enforce",
|
|
4411
|
+
rule: file,
|
|
4412
|
+
reason: action.reason,
|
|
4413
|
+
});
|
|
4517
4414
|
process.stdout.write(JSON.stringify({
|
|
4518
4415
|
hookSpecificOutput: {
|
|
4519
4416
|
hookEventName: on,
|
|
@@ -4523,6 +4420,14 @@ function emitGate(decision, on, mode, file) {
|
|
|
4523
4420
|
}) + "\n");
|
|
4524
4421
|
return;
|
|
4525
4422
|
case "observe":
|
|
4423
|
+
(0, observe_js_1.appendObservation)({
|
|
4424
|
+
kind: "hook",
|
|
4425
|
+
event: on,
|
|
4426
|
+
decision: action.would,
|
|
4427
|
+
mode: "observe",
|
|
4428
|
+
rule: file,
|
|
4429
|
+
reason: action.reason,
|
|
4430
|
+
});
|
|
4526
4431
|
recordObservation(file, on, action.would, action.reason);
|
|
4527
4432
|
console.error(`⚠ [vigiles observe] ${on}: would ${action.would} — ${action.reason}`);
|
|
4528
4433
|
return; // exit 0 — observe never blocks
|
|
@@ -5112,10 +5017,14 @@ async function main() {
|
|
|
5112
5017
|
// Surfaced in the AuditReport (the report's "Create spec" command-emit
|
|
5113
5018
|
// buttons read it) and the terminal nudge below.
|
|
5114
5019
|
const adoptableSurfaces = discoverAdoptableForAudit(root, adapter.layout.instructionFile);
|
|
5020
|
+
// Read the local flight recorder ONCE — feeds both the JSON report
|
|
5021
|
+
// (structured summary, the product boundary) and the terminal render.
|
|
5022
|
+
const ledgerRecords = (0, observe_js_1.readObservations)(root);
|
|
5115
5023
|
const auditReport = (0, audit_report_js_1.buildAuditReport)(report, {
|
|
5116
5024
|
harness: adapter.name,
|
|
5117
5025
|
vigilesVersion: getVersion(),
|
|
5118
5026
|
adoptableSurfaces,
|
|
5027
|
+
observations: (0, observe_js_1.summarizeObservations)(ledgerRecords),
|
|
5119
5028
|
});
|
|
5120
5029
|
const sc = auditReport.score;
|
|
5121
5030
|
const plan = (0, optimize_js_1.optimize)(report);
|
|
@@ -5146,6 +5055,12 @@ async function main() {
|
|
|
5146
5055
|
.length);
|
|
5147
5056
|
if (fireNudge)
|
|
5148
5057
|
console.log("\n" + fireNudge);
|
|
5058
|
+
// The flight recorder: a compact summary of what the harness actually
|
|
5059
|
+
// DID in real sessions (hook/agent decisions), read off the local
|
|
5060
|
+
// agent-readable ledger. Empty (skipped) until something is recorded.
|
|
5061
|
+
const ledgerSummary = (0, observe_js_1.formatLedgerSummary)(ledgerRecords);
|
|
5062
|
+
if (ledgerSummary)
|
|
5063
|
+
console.log("\n" + ledgerSummary);
|
|
5149
5064
|
}
|
|
5150
5065
|
// ONE read-vs-run decision for the EXECUTING checks (live MCP + skill
|
|
5151
5066
|
// firing). A plain `audit` is a deterministic READ; these run only on
|
|
@@ -5179,6 +5094,21 @@ async function main() {
|
|
|
5179
5094
|
// default; `--fail-on-widen` exits non-zero (the opt-in CI gate).
|
|
5180
5095
|
const beforeReport = (0, scan_js_1.scanPlugin)((0, node_path_1.resolve)(capBase), adapter.layout, adapter.dialect);
|
|
5181
5096
|
const diff = (0, capability_diff_js_1.diffCapabilities)(capabilitiesOfReport(beforeReport, adapter.dialect), capabilitiesOfReport(report, adapter.dialect));
|
|
5097
|
+
// Feed the flight recorder: the blast-radius change (moat #2) as a record.
|
|
5098
|
+
// Write to the AUDITED root's ledger (not the caller's cwd) — the same
|
|
5099
|
+
// `root` the audit reads back via `readObservations(root)`, so a
|
|
5100
|
+
// `vigiles audit ./after --capability-diff=./before` from a parent dir
|
|
5101
|
+
// records into ./after/.vigiles/, not the parent workspace.
|
|
5102
|
+
(0, observe_js_1.appendObservation)({
|
|
5103
|
+
kind: "capability-diff",
|
|
5104
|
+
added: [
|
|
5105
|
+
...diff.addedSideEffecting,
|
|
5106
|
+
...diff.addedUnknown,
|
|
5107
|
+
...diff.addedReadOnly,
|
|
5108
|
+
],
|
|
5109
|
+
removed: [...diff.removed],
|
|
5110
|
+
widened: diff.widened,
|
|
5111
|
+
}, root);
|
|
5182
5112
|
console.log(json
|
|
5183
5113
|
? JSON.stringify({ capabilityDiff: diff }, null, 2)
|
|
5184
5114
|
: "\n" + (0, capability_diff_js_1.formatCapabilityDiff)(diff));
|
|
@@ -5277,9 +5207,6 @@ async function main() {
|
|
|
5277
5207
|
}
|
|
5278
5208
|
break;
|
|
5279
5209
|
}
|
|
5280
|
-
case "scaffold-test":
|
|
5281
|
-
handleScaffoldTest(restArgs, args);
|
|
5282
|
-
break;
|
|
5283
5210
|
// --- Plumbing ---
|
|
5284
5211
|
case "generate":
|
|
5285
5212
|
await handleGenerate(restArgs, args);
|
package/dist/eval.js
CHANGED
|
@@ -65,6 +65,7 @@ const node_child_process_1 = require("node:child_process");
|
|
|
65
65
|
const node_fs_1 = require("node:fs");
|
|
66
66
|
const node_os_1 = require("node:os");
|
|
67
67
|
const node_path_1 = require("node:path");
|
|
68
|
+
const observe_js_1 = require("./observe.js");
|
|
68
69
|
const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
|
|
69
70
|
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
70
71
|
const eval_cost_js_1 = require("./eval-cost.js");
|
|
@@ -1677,6 +1678,22 @@ async function measureTriggerRate(spec, opts = {}) {
|
|
|
1677
1678
|
const report = await measureTriggerRateWith(spec, d.runner, d.parse, d.runError, d.harness ?? "claude-code");
|
|
1678
1679
|
// Surface what the run spent (tokens + API-equivalent $ + metered warning).
|
|
1679
1680
|
(0, eval_cost_js_1.emitCostSummary)((0, eval_cost_js_1.costFromArm)(report.usage));
|
|
1681
|
+
// Feed the flight recorder: recall (+ precision when measured) for this skill.
|
|
1682
|
+
const evalName = spec.name ?? "trigger-rate";
|
|
1683
|
+
(0, observe_js_1.appendObservation)({
|
|
1684
|
+
kind: "eval",
|
|
1685
|
+
name: evalName,
|
|
1686
|
+
metric: "recall",
|
|
1687
|
+
value: report.rate,
|
|
1688
|
+
});
|
|
1689
|
+
if (report.precision !== undefined) {
|
|
1690
|
+
(0, observe_js_1.appendObservation)({
|
|
1691
|
+
kind: "eval",
|
|
1692
|
+
name: evalName,
|
|
1693
|
+
metric: "precision",
|
|
1694
|
+
value: report.precision,
|
|
1695
|
+
});
|
|
1696
|
+
}
|
|
1680
1697
|
return report;
|
|
1681
1698
|
}
|
|
1682
1699
|
/* v8 ignore stop */
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/** Bumped when the record shape changes in a non-additive way. */
|
|
2
|
+
export declare const OBSERVE_VERSION = 1;
|
|
3
|
+
/** The ledger filename under the `.vigiles/` directory. */
|
|
4
|
+
export declare const LEDGER_FILE = "runs.jsonl";
|
|
5
|
+
/** Fields every record carries; `v`/`ts` are stamped by the writer, not the caller. */
|
|
6
|
+
export interface ObservationBase {
|
|
7
|
+
/** schema version (`OBSERVE_VERSION`) */
|
|
8
|
+
v: number;
|
|
9
|
+
/** ISO-8601 timestamp */
|
|
10
|
+
ts: string;
|
|
11
|
+
}
|
|
12
|
+
/** A gate/hook decision observed in a real session. */
|
|
13
|
+
export interface HookObservation extends ObservationBase {
|
|
14
|
+
kind: "hook";
|
|
15
|
+
event: string;
|
|
16
|
+
decision: "allow" | "deny" | "ask";
|
|
17
|
+
/** the compiled-hook rule/name, when known */
|
|
18
|
+
rule?: string;
|
|
19
|
+
/** enforce actually blocked; observe recorded a would-be block */
|
|
20
|
+
mode?: "enforce" | "observe";
|
|
21
|
+
/** the bash command inspected, when the gate keyed on one */
|
|
22
|
+
cmd?: string;
|
|
23
|
+
reason?: string;
|
|
24
|
+
}
|
|
25
|
+
/** A subagent tool-contract decision (the PreToolUse rail). */
|
|
26
|
+
export interface AgentObservation extends ObservationBase {
|
|
27
|
+
kind: "agent";
|
|
28
|
+
/** the dispatched subagent */
|
|
29
|
+
name: string;
|
|
30
|
+
tool: string;
|
|
31
|
+
allowed: boolean;
|
|
32
|
+
reason?: string;
|
|
33
|
+
}
|
|
34
|
+
/** Whether a skill fired for a turn (behavioral surface — best-effort per harness). */
|
|
35
|
+
export interface SkillObservation extends ObservationBase {
|
|
36
|
+
kind: "skill";
|
|
37
|
+
name: string;
|
|
38
|
+
fired: boolean;
|
|
39
|
+
}
|
|
40
|
+
/** A measured eval outcome (recall, cost, a check rate, …). */
|
|
41
|
+
export interface EvalObservation extends ObservationBase {
|
|
42
|
+
kind: "eval";
|
|
43
|
+
name: string;
|
|
44
|
+
metric: string;
|
|
45
|
+
value: number;
|
|
46
|
+
}
|
|
47
|
+
/** A capability/blast-radius change observed at PR time. */
|
|
48
|
+
export interface CapabilityDiffObservation extends ObservationBase {
|
|
49
|
+
kind: "capability-diff";
|
|
50
|
+
pr?: number;
|
|
51
|
+
added: string[];
|
|
52
|
+
removed?: string[];
|
|
53
|
+
/** true when the change loosened the agent's effect surface */
|
|
54
|
+
widened: boolean;
|
|
55
|
+
}
|
|
56
|
+
/** The discriminated union every reader narrows on `kind`. */
|
|
57
|
+
export type ObservationRecord = HookObservation | AgentObservation | SkillObservation | EvalObservation | CapabilityDiffObservation;
|
|
58
|
+
/** What a caller supplies — the writer stamps `v` + `ts`. */
|
|
59
|
+
export type ObservationInput = Omit<HookObservation, "v" | "ts"> | Omit<AgentObservation, "v" | "ts"> | Omit<SkillObservation, "v" | "ts"> | Omit<EvalObservation, "v" | "ts"> | Omit<CapabilityDiffObservation, "v" | "ts">;
|
|
60
|
+
/** Serialize one record to a single JSONL line (trailing newline included). */
|
|
61
|
+
export declare function formatObservation(record: ObservationRecord): string;
|
|
62
|
+
/**
|
|
63
|
+
* Append one observation to `<cwd>/.vigiles/runs.jsonl`. Best-effort: any failure is
|
|
64
|
+
* swallowed so recording can never break a live session (the `ts`/`clock` here is a
|
|
65
|
+
* runtime side effect, intentionally — this module records reality, it is not a spec).
|
|
66
|
+
*/
|
|
67
|
+
export declare function appendObservation(input: ObservationInput, cwd?: string): void;
|
|
68
|
+
/**
|
|
69
|
+
* Read the ledger back. Tolerant by design: a malformed or partially-written line is
|
|
70
|
+
* skipped rather than throwing, so a torn append never nukes the whole read.
|
|
71
|
+
*/
|
|
72
|
+
export declare function readObservations(cwd?: string): ObservationRecord[];
|
|
73
|
+
/** Filter the ledger to one record kind, narrowing the type for the caller. */
|
|
74
|
+
export declare function observationsOfKind<K extends ObservationRecord["kind"]>(records: readonly ObservationRecord[], kind: K): Extract<ObservationRecord, {
|
|
75
|
+
kind: K;
|
|
76
|
+
}>[];
|
|
77
|
+
/** A denial rendered as a structured `{label, reason}` — the shared shape the
|
|
78
|
+
* terminal line and the JSON summary both derive from (one-detector-no-drift). */
|
|
79
|
+
export interface LedgerDenial {
|
|
80
|
+
readonly label: string;
|
|
81
|
+
readonly reason: string;
|
|
82
|
+
}
|
|
83
|
+
/** Per-kind record count. */
|
|
84
|
+
export interface LedgerCount {
|
|
85
|
+
readonly kind: ObservationRecord["kind"];
|
|
86
|
+
readonly count: number;
|
|
87
|
+
}
|
|
88
|
+
/** The structured ledger summary carried in the versioned AuditReport JSON. */
|
|
89
|
+
export interface LedgerSummary {
|
|
90
|
+
readonly total: number;
|
|
91
|
+
readonly counts: readonly LedgerCount[];
|
|
92
|
+
readonly denials: number;
|
|
93
|
+
/** The most recent denials (blocked gates / out-of-contract tool calls). */
|
|
94
|
+
readonly recentDenials: readonly LedgerDenial[];
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The structured ledger summary for the AuditReport JSON — total, per-kind counts,
|
|
98
|
+
* and the recent denials. `undefined` when nothing is recorded, so the report field
|
|
99
|
+
* stays absent (additive/optional). Shares `isDenial`/`denialParts` with the
|
|
100
|
+
* terminal `formatLedgerSummary` so the two can't drift.
|
|
101
|
+
*/
|
|
102
|
+
export declare function summarizeObservations(records: readonly ObservationRecord[]): LedgerSummary | undefined;
|
|
103
|
+
/**
|
|
104
|
+
* A compact human summary of the ledger for `vigiles audit` — total, counts by kind,
|
|
105
|
+
* and the recent high-signal denials. Empty string when there is nothing recorded, so
|
|
106
|
+
* the caller can skip the section entirely.
|
|
107
|
+
*/
|
|
108
|
+
export declare function formatLedgerSummary(records: readonly ObservationRecord[]): string;
|
|
109
|
+
//# sourceMappingURL=observe.d.ts.map
|
package/dist/observe.js
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LEDGER_FILE = exports.OBSERVE_VERSION = void 0;
|
|
4
|
+
exports.formatObservation = formatObservation;
|
|
5
|
+
exports.appendObservation = appendObservation;
|
|
6
|
+
exports.readObservations = readObservations;
|
|
7
|
+
exports.observationsOfKind = observationsOfKind;
|
|
8
|
+
exports.summarizeObservations = summarizeObservations;
|
|
9
|
+
exports.formatLedgerSummary = formatLedgerSummary;
|
|
10
|
+
/**
|
|
11
|
+
* The local, agent-readable "flight recorder" ledger — `.vigiles/runs.jsonl`.
|
|
12
|
+
*
|
|
13
|
+
* The connective layer of the four-instrument loop (see the Direction section of
|
|
14
|
+
* CLAUDE.md and `research/harness-observability-direction.md`): every instrument
|
|
15
|
+
* (verify / gate / measure / observe) appends a typed record here, and the file is
|
|
16
|
+
* read three ways off ONE schema — by `vigiles audit`, by the agent debugging its own
|
|
17
|
+
* harness, and (later) by an aggregation surface.
|
|
18
|
+
*
|
|
19
|
+
* Harness-agnostic by construction: the record kinds are neutral concepts, so this
|
|
20
|
+
* lives at the composition/library root (adapters + cli CALL it; it imports no adapter,
|
|
21
|
+
* and it is NOT part of the reference-verification `core/` domain).
|
|
22
|
+
*
|
|
23
|
+
* Append is best-effort — recording must never break a live session.
|
|
24
|
+
*/
|
|
25
|
+
const node_fs_1 = require("node:fs");
|
|
26
|
+
const node_path_1 = require("node:path");
|
|
27
|
+
/** Bumped when the record shape changes in a non-additive way. */
|
|
28
|
+
exports.OBSERVE_VERSION = 1;
|
|
29
|
+
/** The ledger filename under the `.vigiles/` directory. */
|
|
30
|
+
exports.LEDGER_FILE = "runs.jsonl";
|
|
31
|
+
/** Serialize one record to a single JSONL line (trailing newline included). */
|
|
32
|
+
function formatObservation(record) {
|
|
33
|
+
return JSON.stringify(record) + "\n";
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Append one observation to `<cwd>/.vigiles/runs.jsonl`. Best-effort: any failure is
|
|
37
|
+
* swallowed so recording can never break a live session (the `ts`/`clock` here is a
|
|
38
|
+
* runtime side effect, intentionally — this module records reality, it is not a spec).
|
|
39
|
+
*/
|
|
40
|
+
function appendObservation(input, cwd = process.cwd()) {
|
|
41
|
+
try {
|
|
42
|
+
const dir = (0, node_path_1.resolve)(cwd, ".vigiles");
|
|
43
|
+
(0, node_fs_1.mkdirSync)(dir, { recursive: true });
|
|
44
|
+
const record = {
|
|
45
|
+
v: exports.OBSERVE_VERSION,
|
|
46
|
+
ts: new Date().toISOString(),
|
|
47
|
+
...input,
|
|
48
|
+
};
|
|
49
|
+
(0, node_fs_1.appendFileSync)((0, node_path_1.resolve)(dir, exports.LEDGER_FILE), formatObservation(record));
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
/* best-effort — recording is never allowed to break a session */
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Read the ledger back. Tolerant by design: a malformed or partially-written line is
|
|
57
|
+
* skipped rather than throwing, so a torn append never nukes the whole read.
|
|
58
|
+
*/
|
|
59
|
+
function readObservations(cwd = process.cwd()) {
|
|
60
|
+
let raw;
|
|
61
|
+
try {
|
|
62
|
+
raw = (0, node_fs_1.readFileSync)((0, node_path_1.resolve)(cwd, ".vigiles", exports.LEDGER_FILE), "utf8");
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
const out = [];
|
|
68
|
+
for (const line of raw.split("\n")) {
|
|
69
|
+
const trimmed = line.trim();
|
|
70
|
+
if (!trimmed)
|
|
71
|
+
continue;
|
|
72
|
+
const parsed = tryParseRecord(trimmed);
|
|
73
|
+
if (parsed)
|
|
74
|
+
out.push(parsed);
|
|
75
|
+
}
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
/** Filter the ledger to one record kind, narrowing the type for the caller. */
|
|
79
|
+
function observationsOfKind(records, kind) {
|
|
80
|
+
return records.filter((r) => r.kind === kind);
|
|
81
|
+
}
|
|
82
|
+
/** Was this record a denial (a blocked gate or an out-of-contract tool call)? */
|
|
83
|
+
function isDenial(r) {
|
|
84
|
+
return ((r.kind === "hook" && r.decision === "deny") ||
|
|
85
|
+
(r.kind === "agent" && !r.allowed));
|
|
86
|
+
}
|
|
87
|
+
/** Structured description of a denial (the single source for label + reason). */
|
|
88
|
+
function denialParts(r) {
|
|
89
|
+
if (r.kind === "hook")
|
|
90
|
+
return { label: `hook ${r.rule ?? r.event}`, reason: r.reason ?? "denied" };
|
|
91
|
+
if (r.kind === "agent")
|
|
92
|
+
return {
|
|
93
|
+
label: `${r.name} → ${r.tool}`,
|
|
94
|
+
reason: r.reason ?? "outside contract",
|
|
95
|
+
};
|
|
96
|
+
return { label: r.kind, reason: "" };
|
|
97
|
+
}
|
|
98
|
+
/** One line describing a denial for the terminal summary. */
|
|
99
|
+
function denialLine(r) {
|
|
100
|
+
const d = denialParts(r);
|
|
101
|
+
return ` ✗ ${d.label}: ${d.reason}`;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* The structured ledger summary for the AuditReport JSON — total, per-kind counts,
|
|
105
|
+
* and the recent denials. `undefined` when nothing is recorded, so the report field
|
|
106
|
+
* stays absent (additive/optional). Shares `isDenial`/`denialParts` with the
|
|
107
|
+
* terminal `formatLedgerSummary` so the two can't drift.
|
|
108
|
+
*/
|
|
109
|
+
function summarizeObservations(records) {
|
|
110
|
+
if (records.length === 0)
|
|
111
|
+
return undefined;
|
|
112
|
+
const map = new Map();
|
|
113
|
+
for (const r of records)
|
|
114
|
+
map.set(r.kind, (map.get(r.kind) ?? 0) + 1);
|
|
115
|
+
const denied = records.filter(isDenial);
|
|
116
|
+
return {
|
|
117
|
+
total: records.length,
|
|
118
|
+
counts: Array.from(map.entries()).map(([kind, count]) => ({ kind, count })),
|
|
119
|
+
denials: denied.length,
|
|
120
|
+
recentDenials: denied.slice(-5).map(denialParts),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* A compact human summary of the ledger for `vigiles audit` — total, counts by kind,
|
|
125
|
+
* and the recent high-signal denials. Empty string when there is nothing recorded, so
|
|
126
|
+
* the caller can skip the section entirely.
|
|
127
|
+
*/
|
|
128
|
+
function formatLedgerSummary(records) {
|
|
129
|
+
if (records.length === 0)
|
|
130
|
+
return "";
|
|
131
|
+
const lines = [
|
|
132
|
+
`Flight recorder — ${records.length} record${records.length === 1 ? "" : "s"} in .vigiles/${exports.LEDGER_FILE}`,
|
|
133
|
+
];
|
|
134
|
+
const counts = new Map();
|
|
135
|
+
for (const r of records)
|
|
136
|
+
counts.set(r.kind, (counts.get(r.kind) ?? 0) + 1);
|
|
137
|
+
const byKind = Array.from(counts.entries())
|
|
138
|
+
.map(([k, n]) => `${n} ${k}`)
|
|
139
|
+
.join(", ");
|
|
140
|
+
lines.push(` by kind: ${byKind}`);
|
|
141
|
+
const denials = records.filter(isDenial);
|
|
142
|
+
if (denials.length > 0) {
|
|
143
|
+
lines.push(` recent denials (${denials.length}):`);
|
|
144
|
+
for (const r of denials.slice(-5))
|
|
145
|
+
lines.push(denialLine(r));
|
|
146
|
+
}
|
|
147
|
+
return lines.join("\n");
|
|
148
|
+
}
|
|
149
|
+
/** Parse one JSONL line into a record, or `null` if it is not a well-formed record. */
|
|
150
|
+
function tryParseRecord(line) {
|
|
151
|
+
try {
|
|
152
|
+
const value = JSON.parse(line);
|
|
153
|
+
if (value &&
|
|
154
|
+
typeof value === "object" &&
|
|
155
|
+
typeof value.kind === "string") {
|
|
156
|
+
return value;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
/* tolerate a torn/malformed line */
|
|
161
|
+
}
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=observe.js.map
|
package/dist/scaffold-test.js
CHANGED
|
@@ -3,7 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.scaffoldTest = scaffoldTest;
|
|
4
4
|
exports.formatScaffolds = formatScaffolds;
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* The deterministic test-gen engine (B1 v0) — skill-internal (the `test-harness`
|
|
7
|
+
* skill drives it; there is no standalone CLI verb).
|
|
7
8
|
*
|
|
8
9
|
* Free-form in, a RUNNABLE starter test out. Given an existing hand-written
|
|
9
10
|
* skill / subagent / hook, emit a scaffolded `*.harness.mjs` / `*.eval.mjs` at the
|
|
@@ -35,7 +36,7 @@ function header(title, run) {
|
|
|
35
36
|
"/**",
|
|
36
37
|
` * ${title}`,
|
|
37
38
|
" *",
|
|
38
|
-
" * Generated by
|
|
39
|
+
" * Generated by vigiles (the test-harness skill) — a STARTER, not a finished test. Fill in",
|
|
39
40
|
" * the TODOs (they're where a human/model must supply judgement), then run:",
|
|
40
41
|
` * ${run}`,
|
|
41
42
|
" */",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "12.
|
|
3
|
+
"version": "12.2.0",
|
|
4
4
|
"description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|