skilldrop-cli 0.16.27 β 0.16.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/governance.js +63 -2
- package/bin/skilldrop.js +65 -2
- package/contracts/policy.schema.json +59 -11
- package/package.json +1 -1
- package/packs/data-analytics/loops/measure/LOOP.md +118 -0
- package/packs/data-analytics/loops/measure/evals/eval_queries.json +9 -0
- package/packs/data-analytics/loops/measure/evals/evals.json +28 -0
- package/packs/data-analytics/loops/measure/loop.json +41 -0
- package/packs/grc/loops/comply/LOOP.md +133 -0
- package/packs/grc/loops/comply/evals/eval_queries.json +9 -0
- package/packs/grc/loops/comply/evals/evals.json +30 -0
- package/packs/grc/loops/comply/loop.json +54 -0
- package/packs/research/loops/research/LOOP.md +123 -0
- package/packs/research/loops/research/evals/eval_queries.json +9 -0
- package/packs/research/loops/research/evals/evals.json +39 -0
- package/packs/research/loops/research/loop.json +35 -0
package/bin/governance.js
CHANGED
|
@@ -16,7 +16,7 @@ const { execFileSync } = require("child_process");
|
|
|
16
16
|
const SEVERITY = { "π₯": 3, "π§": 2, "π¨": 1 }; // scan findings; βͺ (fixtures) never count
|
|
17
17
|
const MAX = { none: 0, low: 1, medium: 2, high: 3 }; // policy scan.max: the worst finding allowed
|
|
18
18
|
const KEYS = {
|
|
19
|
-
"": ["version", "catalogues", "skills", "scan", "audit", "note"],
|
|
19
|
+
"": ["version", "catalogues", "skills", "scan", "audit", "require_containment", "note"],
|
|
20
20
|
catalogues: ["allow", "require_pinned"],
|
|
21
21
|
skills: ["allow", "block"],
|
|
22
22
|
scan: ["max"],
|
|
@@ -60,6 +60,8 @@ function load(flags) {
|
|
|
60
60
|
list(p.skills && p.skills.block, "skills.block");
|
|
61
61
|
if (p.catalogues && p.catalogues.require_pinned !== undefined && typeof p.catalogues.require_pinned !== "boolean")
|
|
62
62
|
problems.push("catalogues.require_pinned must be true or false");
|
|
63
|
+
if (p.require_containment !== undefined && typeof p.require_containment !== "boolean")
|
|
64
|
+
problems.push("require_containment must be true or false");
|
|
63
65
|
if (p.scan && p.scan.max !== undefined && !(p.scan.max in MAX)) problems.push(`scan.max must be one of ${Object.keys(MAX).join(", ")}`);
|
|
64
66
|
if (p.audit) {
|
|
65
67
|
const a = p.audit.actor;
|
|
@@ -209,4 +211,63 @@ function verifyMirror(dir) {
|
|
|
209
211
|
return { source: m.source, created: m.created, files: Object.keys(want).length, missing, changed, extra };
|
|
210
212
|
}
|
|
211
213
|
|
|
212
|
-
|
|
214
|
+
/* ---------- containment (RFC-0051, OWASP AST06) ----------
|
|
215
|
+
Claude Code's sandbox confines the shell commands a session runs, and every process they
|
|
216
|
+
start: writes to the working directory and temp, network only to allowed hosts. It applies
|
|
217
|
+
to a whole project session, not to one skill, so containing a skill means turning the strict
|
|
218
|
+
sandbox on for the project it is installed into, sized to what its manifest declares
|
|
219
|
+
(permissions.network, permissions.files). It only ever adds restrictions: it never removes
|
|
220
|
+
a setting, never sets one to something looser, and never widens a sandbox that is already
|
|
221
|
+
on. Facts from code.claude.com/docs/en/sandboxing and /permissions, checked 2026-10-19. */
|
|
222
|
+
function readSettings(f) {
|
|
223
|
+
if (!f || !fs.existsSync(f)) return {};
|
|
224
|
+
try { return JSON.parse(fs.readFileSync(f, "utf8")); }
|
|
225
|
+
catch (e) { die(`${tilde(f)} is not valid JSON β fix it before containment changes it; nothing was installed`); }
|
|
226
|
+
}
|
|
227
|
+
const strictIn = (d) => !!(d && d.sandbox && d.sandbox.enabled === true && d.sandbox.allowUnsandboxedCommands === false);
|
|
228
|
+
|
|
229
|
+
/* units: [{name, perms}]; files: {local, project, user}; opts: {require}.
|
|
230
|
+
Returns what --contain would change, without writing anything. */
|
|
231
|
+
function containmentPlan(units, files, opts = {}) {
|
|
232
|
+
const uncontainable = units.filter((u) => u.perms && u.perms.files === "anywhere").map((u) => u.name);
|
|
233
|
+
const hosts = [...new Set(units.flatMap((u) => (u.perms && u.perms.network) || []))].sort();
|
|
234
|
+
const local = readSettings(files.local);
|
|
235
|
+
const others = [files.project, files.user].filter((f) => f && f !== files.local).map(readSettings);
|
|
236
|
+
const before = JSON.stringify(local);
|
|
237
|
+
const next = JSON.parse(before);
|
|
238
|
+
const sb = (next.sandbox = next.sandbox && typeof next.sandbox === "object" ? next.sandbox : {});
|
|
239
|
+
const alreadyOn = sb.enabled === true || others.some((d) => d.sandbox && d.sandbox.enabled === true);
|
|
240
|
+
const changes = [], notes = [];
|
|
241
|
+
if (sb.enabled !== true) { sb.enabled = true; changes.push("sandbox.enabled: true β shell commands and what they start run in the OS sandbox"); }
|
|
242
|
+
if (sb.allowUnsandboxedCommands !== false) {
|
|
243
|
+
sb.allowUnsandboxedCommands = false;
|
|
244
|
+
changes.push("sandbox.allowUnsandboxedCommands: false β a command that fails in the sandbox can't retry outside it");
|
|
245
|
+
}
|
|
246
|
+
if (opts.require && sb.failIfUnavailable !== true) {
|
|
247
|
+
sb.failIfUnavailable = true;
|
|
248
|
+
changes.push("sandbox.failIfUnavailable: true β Claude Code won't start here if the sandbox can't (the policy requires containment)");
|
|
249
|
+
}
|
|
250
|
+
if (hosts.length && alreadyOn)
|
|
251
|
+
notes.push(`these skills declare network hosts (${hosts.join(", ")}). The sandbox was already on, so they were not added: ` +
|
|
252
|
+
"adding hosts would widen it. Add them to sandbox.network.allowedDomains yourself if the skills should reach them.");
|
|
253
|
+
else if (hosts.length) {
|
|
254
|
+
sb.network = sb.network && typeof sb.network === "object" ? sb.network : {};
|
|
255
|
+
const have = new Set(sb.network.allowedDomains || []);
|
|
256
|
+
const add = hosts.filter((h) => !have.has(h));
|
|
257
|
+
if (add.length) {
|
|
258
|
+
sb.network.allowedDomains = [...(sb.network.allowedDomains || []), ...add];
|
|
259
|
+
changes.push(`sandbox.network.allowedDomains: ${add.join(", ")} β the hosts these skills declare; shell commands reach no other host without asking`);
|
|
260
|
+
}
|
|
261
|
+
} else if (!alreadyOn) notes.push("these skills declare no network hosts, so shell commands reach none without asking");
|
|
262
|
+
if ((sb.excludedCommands || []).length)
|
|
263
|
+
notes.push(`this file's sandbox.excludedCommands (${sb.excludedCommands.join(", ")}) still run outside the sandbox`);
|
|
264
|
+
notes.push('the sandbox covers shell commands only: Claude Code\'s file tools, web tools, MCP servers and hooks follow permission rules instead. ' +
|
|
265
|
+
'To keep these skills off the web entirely, add "permissions": {"deny": ["WebFetch", "WebSearch"]} to this file.');
|
|
266
|
+
return { uncontainable, hosts, changes, notes, next, changed: JSON.stringify(next) !== before };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/* Is a project already contained (strict sandbox on in its own settings)? For inventory and
|
|
270
|
+
for a policy that requires containment. */
|
|
271
|
+
function contained(files) { return strictIn(readSettings(files.local)) || strictIn(readSettings(files.project)); }
|
|
272
|
+
|
|
273
|
+
module.exports = { containmentPlan, contained, load, check, prepare, audit, verifyAudit, verifyMirror, sourceKey, tilde, SEVERITY };
|
package/bin/skilldrop.js
CHANGED
|
@@ -105,6 +105,9 @@ Install/update/uninstall targets (pick one):
|
|
|
105
105
|
repo you don't own without touching its tracked files or .gitignore
|
|
106
106
|
|
|
107
107
|
Organisation policy (RFC-0050):
|
|
108
|
+
--contain Claude Code project installs: turn on the strict sandbox for this project in
|
|
109
|
+
.claude/settings.local.json, allowing only the network hosts the skills declare.
|
|
110
|
+
Only adds restrictions; refuses a skill that declares writing anywhere (RFC-0051)
|
|
108
111
|
--policy <file> the policy to enforce; also SKILLDROP_POLICY, else ~/.config/skilldrop/policy.json.
|
|
109
112
|
It can allow only some catalogues (and require them pinned), allow or block skills,
|
|
110
113
|
cap the supply-chain scan's findings, and turn on an audit log. A policy that is
|
|
@@ -459,6 +462,62 @@ function enforcePolicy(flags, cat, names, kind) {
|
|
|
459
462
|
return policy;
|
|
460
463
|
}
|
|
461
464
|
|
|
465
|
+
/* RFC-0051: --contain turns on Claude Code's strict sandbox for the project a skill is installed
|
|
466
|
+
into, sized to what the skills declare. Planned (and refused) before anything is copied;
|
|
467
|
+
applied after. Only ever adds restrictions; see gov.containmentPlan. */
|
|
468
|
+
function containmentFor(flags, cat, names, dest, ide, policy) {
|
|
469
|
+
const require = !!(policy && policy.require_containment);
|
|
470
|
+
if (!flags.contain && !require) return null;
|
|
471
|
+
if (ide !== "claude") {
|
|
472
|
+
if (require) die(`the policy requires containment, which skilldrop sets up for Claude Code project installs; ` +
|
|
473
|
+
`${ide === "generic" ? "--dest" : "--ide " + ide} has no sandbox it can configure β nothing was installed`);
|
|
474
|
+
const want = names.map((s) => { const pm = manifestOf(cat, s).permissions; return pm ? `${s}: network ${(pm.network || []).join(", ") || "none"}, writes ${pm.files}` : `${s}: no scripts`; });
|
|
475
|
+
console.log(`note: ${ide === "generic" ? "this target" : ide} has no sandbox skilldrop can configure. What the skills expect:\n ${want.join("\n ")}`);
|
|
476
|
+
return null;
|
|
477
|
+
}
|
|
478
|
+
const userDir = path.join(os.homedir(), ".claude", "skills");
|
|
479
|
+
if (path.resolve(dest) === userDir)
|
|
480
|
+
die("containment applies to one project: install with --project --contain (or --local --contain). A user-scope " +
|
|
481
|
+
"install reaches every project, and skilldrop won't change your user settings β nothing was installed");
|
|
482
|
+
const base = path.dirname(dest);
|
|
483
|
+
const files = { local: path.join(base, "settings.local.json"), project: path.join(base, "settings.json"),
|
|
484
|
+
user: path.join(os.homedir(), ".claude", "settings.json") };
|
|
485
|
+
if (!flags.contain) {
|
|
486
|
+
if (gov.contained(files)) return null;
|
|
487
|
+
die("the policy requires containment: re-run with --contain, which turns on Claude Code's strict sandbox for this " +
|
|
488
|
+
"project β nothing was installed");
|
|
489
|
+
}
|
|
490
|
+
const plan = gov.containmentPlan(names.map((s) => ({ name: s, perms: manifestOf(cat, s).permissions })), files, { require });
|
|
491
|
+
if (plan.uncontainable.length)
|
|
492
|
+
die(`${plan.uncontainable.join(", ")} declare${plan.uncontainable.length === 1 ? "s" : ""} writing anywhere on disk ` +
|
|
493
|
+
"(permissions.files), so it can't run inside the sandbox. Install without --contain, or leave it out β nothing was installed");
|
|
494
|
+
return Object.assign(plan, { files });
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function applyContainment(plan, dry) {
|
|
498
|
+
const f = plan.files.local;
|
|
499
|
+
const where = path.relative(process.cwd(), f) || f;
|
|
500
|
+
console.log(`\n${dry ? "would contain" : "Contained"} (RFC-0051): ${where}`);
|
|
501
|
+
if (!plan.changed) console.log(" already contained: nothing to change");
|
|
502
|
+
for (const c of plan.changes) console.log(` ${dry ? "would set" : "set"} ${c}`);
|
|
503
|
+
for (const n of plan.notes) console.log(` note: ${n}`);
|
|
504
|
+
if (dry) return;
|
|
505
|
+
if (plan.changed) {
|
|
506
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
507
|
+
fs.writeFileSync(f, JSON.stringify(plan.next, null, 2) + "\n");
|
|
508
|
+
}
|
|
509
|
+
// settings.local.json is personal: keep it out of git if nothing ignores it already
|
|
510
|
+
const root = gitRoot(process.cwd());
|
|
511
|
+
if (root && path.resolve(f).startsWith(path.resolve(root) + path.sep)) {
|
|
512
|
+
let ignored = false;
|
|
513
|
+
try { execFileSync("git", ["check-ignore", "-q", f], { cwd: root, stdio: "ignore" }); ignored = true; } catch (e) { /* not ignored */ }
|
|
514
|
+
if (!ignored) {
|
|
515
|
+
writeExclude(root, readExclude(root).entries.concat(excludeEntry(root, f, false)));
|
|
516
|
+
console.log(` listed it in ${path.relative(process.cwd(), excludePath(root)) || excludePath(root)} so git won't see it`);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
462
521
|
const auditUnit = (cat, name, kind, version, dest) =>
|
|
463
522
|
({ kind, name, version: version || null, source: cat.source, commit: cat.commit || null, pinned: !!cat.pinned, target: dest });
|
|
464
523
|
|
|
@@ -1265,7 +1324,8 @@ function install(args) {
|
|
|
1265
1324
|
gate(cat, names);
|
|
1266
1325
|
const policy = enforcePolicy(args.flags, cat, names, "skill");
|
|
1267
1326
|
const { dest, ide } = target(args.flags);
|
|
1268
|
-
|
|
1327
|
+
const contain = containmentFor(args.flags, cat, names, dest, ide, policy);
|
|
1328
|
+
if (args.flags["dry-run"]) { planInstall(cat, names, dest, ide, args.flags); if (contain) applyContainment(contain, true); return; }
|
|
1269
1329
|
fs.mkdirSync(dest, { recursive: true });
|
|
1270
1330
|
const l = ledger(dest);
|
|
1271
1331
|
const pipDeps = [], suggestions = new Set(), notes = [];
|
|
@@ -1276,7 +1336,8 @@ function install(args) {
|
|
|
1276
1336
|
console.log(`installed ${s}@${m.version} -> ${dest}`);
|
|
1277
1337
|
}
|
|
1278
1338
|
saveLedger(l);
|
|
1279
|
-
|
|
1339
|
+
if (contain) applyContainment(contain, false);
|
|
1340
|
+
gov.audit(policy, "install", names.map((s) => Object.assign(auditUnit(cat, s, "skill", lver(l.data[s]), dest), contain ? { contained: true } : {})));
|
|
1280
1341
|
console.log(`\n${names.length} skill(s) installed (${ide}).`);
|
|
1281
1342
|
if (notes.length) console.log(`\nCleanup:\n${notes.join("\n")}`);
|
|
1282
1343
|
const local = !!args.flags.local;
|
|
@@ -2253,6 +2314,8 @@ function inventory(args) {
|
|
|
2253
2314
|
installs.push({ target: gov.tilde(t.dest), scope: t.scope, ide: t.ide, kind, name, version: lver(e) || null,
|
|
2254
2315
|
source: gov.tilde(src), commit: (e && e.commit) || null, pinned,
|
|
2255
2316
|
edited: kind === "agent" ? 0 : editedFiles(t.dest, name, e).length,
|
|
2317
|
+
contained: t.ide === "claude" && t.scope === "project" && kind !== "agent"
|
|
2318
|
+
? gov.contained({ local: path.join(path.dirname(t.dest), "settings.local.json"), project: path.join(path.dirname(t.dest), "settings.json") }) : null,
|
|
2256
2319
|
policy: policy ? (refused.length ? refused.map((x) => x.why) : "ok") : null });
|
|
2257
2320
|
}
|
|
2258
2321
|
}
|
|
@@ -5,31 +5,67 @@
|
|
|
5
5
|
"$comment": "RFC-0050. The file an organisation hands out (--policy, SKILLDROP_POLICY, or ~/.config/skilldrop/policy.json). Closed at every level: an unknown key stops the command, because a misspelt rule that is silently ignored is a rule that isn't there. bin/governance.js enforces the same shape; contracts/policy.example.json must satisfy both.",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"additionalProperties": false,
|
|
8
|
-
"required": [
|
|
8
|
+
"required": [
|
|
9
|
+
"version"
|
|
10
|
+
],
|
|
9
11
|
"properties": {
|
|
10
|
-
"version": {
|
|
11
|
-
|
|
12
|
+
"version": {
|
|
13
|
+
"const": 1
|
|
14
|
+
},
|
|
15
|
+
"note": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"description": "Free text: who owns this policy and where to ask about it."
|
|
18
|
+
},
|
|
12
19
|
"catalogues": {
|
|
13
20
|
"type": "object",
|
|
14
21
|
"additionalProperties": false,
|
|
15
22
|
"properties": {
|
|
16
|
-
"allow": {
|
|
17
|
-
|
|
23
|
+
"allow": {
|
|
24
|
+
"type": "array",
|
|
25
|
+
"items": {
|
|
26
|
+
"type": "string"
|
|
27
|
+
},
|
|
28
|
+
"description": "Catalogue sources allowed: \"bundled\", a git URL (a trailing * matches a prefix, e.g. https://git.example.org/skills/*), or a path. Absent: any."
|
|
29
|
+
},
|
|
30
|
+
"require_pinned": {
|
|
31
|
+
"type": "boolean",
|
|
32
|
+
"description": "Every catalogue other than the bundled one must be pinned to a commit (--from <url>#<commit>)."
|
|
33
|
+
}
|
|
18
34
|
}
|
|
19
35
|
},
|
|
20
36
|
"skills": {
|
|
21
37
|
"type": "object",
|
|
22
38
|
"additionalProperties": false,
|
|
23
39
|
"properties": {
|
|
24
|
-
"allow": {
|
|
25
|
-
|
|
40
|
+
"allow": {
|
|
41
|
+
"type": "array",
|
|
42
|
+
"items": {
|
|
43
|
+
"type": "string"
|
|
44
|
+
},
|
|
45
|
+
"description": "Skill, loop and agent names allowed; * matches anything. Absent: any."
|
|
46
|
+
},
|
|
47
|
+
"block": {
|
|
48
|
+
"type": "array",
|
|
49
|
+
"items": {
|
|
50
|
+
"type": "string"
|
|
51
|
+
},
|
|
52
|
+
"description": "Names refused even when allowed."
|
|
53
|
+
}
|
|
26
54
|
}
|
|
27
55
|
},
|
|
28
56
|
"scan": {
|
|
29
57
|
"type": "object",
|
|
30
58
|
"additionalProperties": false,
|
|
31
59
|
"properties": {
|
|
32
|
-
"max": {
|
|
60
|
+
"max": {
|
|
61
|
+
"enum": [
|
|
62
|
+
"none",
|
|
63
|
+
"low",
|
|
64
|
+
"medium",
|
|
65
|
+
"high"
|
|
66
|
+
],
|
|
67
|
+
"description": "The worst supply-chain scan finding a skill may carry: none, low (π¨), medium (π§), high (π₯). Findings in eval fixtures and example inputs never count."
|
|
68
|
+
}
|
|
33
69
|
}
|
|
34
70
|
},
|
|
35
71
|
"audit": {
|
|
@@ -37,10 +73,22 @@
|
|
|
37
73
|
"additionalProperties": false,
|
|
38
74
|
"description": "Present: install, update and uninstall append to a tamper-evident log.",
|
|
39
75
|
"properties": {
|
|
40
|
-
"log": {
|
|
41
|
-
|
|
42
|
-
|
|
76
|
+
"log": {
|
|
77
|
+
"type": "string",
|
|
78
|
+
"description": "Where the log goes. Default ~/.config/skilldrop/audit.jsonl. Refused inside a git working tree."
|
|
79
|
+
},
|
|
80
|
+
"actor": {
|
|
81
|
+
"description": "Who each entry names: \"pseudonym\" (default; a keyed hash of user and machine, needs the salt), \"none\", \"os-user\", or {\"fixed\": \"<name>\"} such as a team or machine id."
|
|
82
|
+
},
|
|
83
|
+
"salt_env": {
|
|
84
|
+
"type": "string",
|
|
85
|
+
"description": "The environment variable holding the organisation's secret salt for pseudonyms. Default SKILLDROP_AUDIT_SALT."
|
|
86
|
+
}
|
|
43
87
|
}
|
|
88
|
+
},
|
|
89
|
+
"require_containment": {
|
|
90
|
+
"type": "boolean",
|
|
91
|
+
"description": "RFC-0051. true: an install must be contained (--contain, or a Claude Code project whose own settings already run the strict sandbox); targets that can't be are refused, and --contain also sets sandbox.failIfUnavailable."
|
|
44
92
|
}
|
|
45
93
|
}
|
|
46
94
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "skilldrop-cli",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.29",
|
|
4
4
|
"description": "Portable AI-agent skills, measured against the agent without them. ADRs, PRDs, runbooks, decks and reviews, installable into Claude Code, Cursor, Kiro, and any IDE that takes custom instructions. One command per skill or per role-based pack.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Sanjay Ananthanarayan",
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: measure
|
|
3
|
+
description: Take a dashboard request to a spec a BI developer can build on numbers people can defend: the decision it serves and one chart per question, every metric defined, the SQL behind each metric reviewed for the bugs that quietly inflate a figure, and the metric owner's sign-off on each definition before anything is built. Use when the user says "we need a dashboard forβ¦", "run the measure loop", "make sure these numbers are right before the BI team builds it", or two teams disagree about a number that is about to go on a dashboard.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# measure
|
|
7
|
+
|
|
8
|
+
One dashboard, from the request to a spec whose numbers have been checked. A wrong number
|
|
9
|
+
on a dashboard does more damage than a wrong number in a query, because the chart makes it
|
|
10
|
+
look settled. So the loop reviews the SQL before anything is built, and it only reviews the
|
|
11
|
+
queries that matter. The spec runs first and cuts the charts that serve no decision, so no
|
|
12
|
+
time goes on queries for charts that won't exist. Last, each metric's owner confirms the
|
|
13
|
+
definition, because a query can be correct and still compute the wrong thing.
|
|
14
|
+
|
|
15
|
+
## Stages
|
|
16
|
+
|
|
17
|
+
| # | Stage | Type | Skills | Gate |
|
|
18
|
+
|---|---|---|---|---|
|
|
19
|
+
| 1 | `spec` | generate | [`dashboard-spec`](../../skills/dashboard-spec/SKILL.md) | β |
|
|
20
|
+
| 2 | `review` | verify | [`sql-review`](../../skills/sql-review/SKILL.md) | **G9** review |
|
|
21
|
+
| 3 | `sign-off` | gate | [`doc-critique`](../../../core/skills/doc-critique/SKILL.md) | **G9.1** human |
|
|
22
|
+
|
|
23
|
+
## How to run this loop
|
|
24
|
+
|
|
25
|
+
1. **Spec the dashboard.** Run `dashboard-spec` with the audience, the decision and the
|
|
26
|
+
charts people asked for. Save `dashboard-spec.md`. Its cut list removes charts before any
|
|
27
|
+
SQL is read. Every surviving chart names its metric by defined name, grain, window and
|
|
28
|
+
comparison. A metric with no written definition is marked `undefined`. Write it down as
|
|
29
|
+
the person states it, or leave it as a question for the metric's owner. Don't define it
|
|
30
|
+
yourself.
|
|
31
|
+
2. **Collect one query per surviving metric.** The person supplies the SQL, the dbt model or
|
|
32
|
+
the view that computes each one, with the engine and what one output row represents.
|
|
33
|
+
Where none exists yet, draft one from the definition and the schema they supply, and
|
|
34
|
+
mark it as a draft. A metric with no query and no schema to draft from is `BLOCKED` for
|
|
35
|
+
that metric, not skipped.
|
|
36
|
+
3. **Review every query at G9.** Run `sql-review` on each, one at a time. Collect each
|
|
37
|
+
review's verdict line, findings, corrected query and verification queries. Ask the person
|
|
38
|
+
to run the verification queries, or run them yourself when you have access.
|
|
39
|
+
4. **Map the reviews to G9's verdict:**
|
|
40
|
+
- every query `SHIP IT` or `SHIP WITH NOTES` β `READY`. The notes travel to the owner at G9.1.
|
|
41
|
+
- any query `BLOCK` or `FIX FIRST` β `NOT READY`. Return to `spec`. The corrected query
|
|
42
|
+
becomes that metric's source in the spec, and its verification results go beside it.
|
|
43
|
+
The fix changes the number the chart will show, sometimes by a lot, so its target,
|
|
44
|
+
comparison and any threshold the decision uses need checking against it. Then review
|
|
45
|
+
the corrected query again. Each return counts against the cap.
|
|
46
|
+
5. **Critique, then put each definition to its owner at G9.1.** Run `doc-critique` on the
|
|
47
|
+
spec. Then show each metric's owner:
|
|
48
|
+
- the definition in words;
|
|
49
|
+
- the reviewed query;
|
|
50
|
+
- what changed and by how much, from the verification queries;
|
|
51
|
+
- every assumption the review made about keys, time zones and exclusions.
|
|
52
|
+
|
|
53
|
+
The owner confirms that this is what the business means by the metric. The BI developer
|
|
54
|
+
builds only from a spec that has passed G9.1.
|
|
55
|
+
|
|
56
|
+
## Verdicts
|
|
57
|
+
|
|
58
|
+
Defined in [`contracts/terminals.json`](../../../../contracts/terminals.json).
|
|
59
|
+
|
|
60
|
+
- G9 emits `READY`, `NOT READY` or `BLOCKED` (no query text, or no engine to review it
|
|
61
|
+
against).
|
|
62
|
+
- G9.1 emits:
|
|
63
|
+
- `PROCEED`: every metric's owner confirmed its definition and number.
|
|
64
|
+
- `PROCEED WITH CONDITIONS`: confirmed, with named items that travel to the build, such as
|
|
65
|
+
a `SHIP WITH NOTES` caveat shown on the chart, or a data gap with its date.
|
|
66
|
+
- `REVISE`: an owner says the definition is wrong (the wrong time zone, an exclusion
|
|
67
|
+
missing, the wrong grain). Return to `spec` with their definition.
|
|
68
|
+
- `BLOCKED`: an `undefined` metric has no owner to define it, or an owner is unavailable.
|
|
69
|
+
|
|
70
|
+
An `undefined` metric can't pass G9.1. Its owner defines it, or the chart moves to the cut
|
|
71
|
+
list with the reason.
|
|
72
|
+
|
|
73
|
+
## Non-interactive runs
|
|
74
|
+
|
|
75
|
+
With no person present, G9.1 cannot be satisfied. Run `spec` and `review`, deliver the spec
|
|
76
|
+
with each query's review, and emit `BLOCKED: need each metric owner's confirmation of
|
|
77
|
+
<every metric on the spec>`. A metric whose query passed G9 still needs its owner: the review
|
|
78
|
+
says the SQL computes the definition, not that the definition is right. Never confirm a definition on an owner's behalf, and never fill an
|
|
79
|
+
`undefined` metric with a plausible one: two teams will each read it as theirs.
|
|
80
|
+
|
|
81
|
+
## When a stage's skill is not installed
|
|
82
|
+
|
|
83
|
+
Name the missing skill and skilldrop as its source, do the minimal inline version of that
|
|
84
|
+
stage, and record that it ran degraded. A degraded `review` still checks every join for
|
|
85
|
+
fan-out, every filter for NULLs and every date boundary for its time zone, in that order,
|
|
86
|
+
because those are the bugs that change the number. Never skip G9.1.
|
|
87
|
+
|
|
88
|
+
## Run log (opt-in)
|
|
89
|
+
|
|
90
|
+
Only when the environment variable `SKILLDROP_LOOP_LOG` is set to a file path: after each
|
|
91
|
+
gate's verdict, append one JSON line to that file. Nothing else is recorded, and nothing is
|
|
92
|
+
sent anywhere. When the variable isn't set, or you can't read it, skip this section.
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
printf '%s\n' '{"ts":"2026-10-19T14:03:00Z","loop":"measure","stage":"review","gate":"G9","verdict":"NOT READY","round":1}' >> "$SKILLDROP_LOOP_LOG"
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Fields: `ts` (UTC), `loop`, `stage`, `gate` (`null` for a stage without one), `verdict` (the
|
|
99
|
+
gate's exact word), and `round` (1 for the first attempt, counting returns under the cap).
|
|
100
|
+
The log holds verdicts only: never a query, a table name or a figure.
|
|
101
|
+
`skilldrop loop-stats` summarises the file.
|
|
102
|
+
|
|
103
|
+
## Quality bar
|
|
104
|
+
|
|
105
|
+
- **The cut list runs before any SQL is read.** Only the queries behind charts that serve the decision get reviewed.
|
|
106
|
+
- **Every surviving chart's metric has a definition and a reviewed query.** `undefined` and unreviewed are build blockers, not notes.
|
|
107
|
+
- **A fixed query goes back through the spec.** The corrected number can move a target or a threshold, and the spec is where those live.
|
|
108
|
+
- **The owner sees the change in the number.** The verification queries' before and after go to G9.1, not only the corrected SQL.
|
|
109
|
+
- **The metric's owner signs the definition.** A correct query for the wrong definition is still the wrong number.
|
|
110
|
+
|
|
111
|
+
## Anti-patterns to avoid
|
|
112
|
+
|
|
113
|
+
- β **Reviewing every query in the BI tool.** Queries for charts the spec cut cost review time and add nothing.
|
|
114
|
+
- β **Defining an `undefined` metric yourself to finish the spec.** That settles a disagreement between two teams by picking one at random.
|
|
115
|
+
- β **Shipping the corrected SQL without telling the owner the number moved.** A revenue figure that drops a third overnight with no explanation gets reverted.
|
|
116
|
+
- β **Passing G9 on a clean lint.** The lint can't see join fan-out; the review reads the joins.
|
|
117
|
+
- β **Treating `SHIP WITH NOTES` as `SHIP IT`.** The notes go to the owner and, where they change how the chart reads, onto the chart.
|
|
118
|
+
- β **Building before G9.1.** A dashboard built on an unconfirmed definition is the one the two teams will argue about.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"query": "we need a weekly revenue dashboard for the regional leads, and the numbers have to be right", "should_trigger": true},
|
|
3
|
+
{"query": "run the measure loop on the support KPI dashboard request", "should_trigger": true},
|
|
4
|
+
{"query": "make sure these numbers are right before the BI team builds the exec dashboard", "should_trigger": true},
|
|
5
|
+
{"query": "finance and sales disagree on active customers and it's going on the board dashboard", "should_trigger": true},
|
|
6
|
+
{"query": "is this SQL query right?", "should_trigger": false},
|
|
7
|
+
{"query": "what charts should go on our marketing dashboard?", "should_trigger": false},
|
|
8
|
+
{"query": "define the success metric for the new onboarding flow", "should_trigger": false}
|
|
9
|
+
]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"loop_name": "measure",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": "spec-first-then-review",
|
|
6
|
+
"prompt": "Run the measure loop. Regional sales leads want a dashboard to decide each Monday where to send the two floating account managers. They asked for: weekly revenue by region, revenue all time, website sessions, active customers, average deal size by region, and a map. Revenue's query is in revenue.sql (Postgres; one row per region per week; customers.customer_id is unique and an order has many order_items). Nobody has written down what 'active customer' means; sales counts anyone with an open opportunity, finance counts anyone invoiced in 90 days.",
|
|
7
|
+
"assertions": [
|
|
8
|
+
"Runs dashboard-spec first and writes the decision statement before reviewing any SQL",
|
|
9
|
+
"Cuts all-time revenue, website sessions and the map with reasons",
|
|
10
|
+
"Marks active customers as undefined and does not choose between the sales and finance definitions",
|
|
11
|
+
"Reviews revenue.sql and flags the order_items join as multiplying revenue",
|
|
12
|
+
"Reports G9 as NOT READY and returns to spec with the corrected query as the metric's source",
|
|
13
|
+
"Lists verification queries that show how much the revenue number changes",
|
|
14
|
+
"Stops at G9.1 asking the metric owners to confirm; does not confirm a definition itself"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"id": "owner-rejects-the-definition",
|
|
19
|
+
"prompt": "Continuing the measure loop. The revenue owner, finance, says the corrected query is right but revenue must be booked in New York time and exclude refunded orders. The query uses UTC and doesn't look at refunds.",
|
|
20
|
+
"assertions": [
|
|
21
|
+
"Treats this as G9.1 REVISE and returns to spec with finance's definition",
|
|
22
|
+
"Updates the revenue metric's definition in the spec to New York time and excluding refunded orders",
|
|
23
|
+
"Reviews the changed query again at G9 before returning to sign-off",
|
|
24
|
+
"Counts the return under the cap of 3"
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "RFC-0052. A dashboard request to a spec whose every number is defined and whose every query has been reviewed. The spec comes first so only the queries behind charts that survive the cut list are reviewed; the metric's owner signs last, because only they can say a definition is what the business means.",
|
|
3
|
+
"name": "measure",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"description": "Take a dashboard request to a spec a BI developer can build on numbers people can defend: the decision it serves and one chart per question, every metric defined, the SQL behind each metric reviewed for the bugs that quietly inflate a figure, and the metric owner's sign-off on each definition before anything is built. Use when the user says \"we need a dashboard forβ¦\", \"run the measure loop\", \"make sure these numbers are right before the BI team builds it\", or two teams disagree about a number that is about to go on a dashboard.",
|
|
6
|
+
"entrypoint": "LOOP.md",
|
|
7
|
+
"kind": "loop",
|
|
8
|
+
"cap": 3,
|
|
9
|
+
"stages": [
|
|
10
|
+
{
|
|
11
|
+
"id": "spec",
|
|
12
|
+
"type": "generate",
|
|
13
|
+
"intent": "Pin the audience and the decision, keep only the charts that serve it, and name the defined metric behind each, flagging any metric nobody has defined.",
|
|
14
|
+
"skills": ["dashboard-spec"]
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "review",
|
|
18
|
+
"type": "verify",
|
|
19
|
+
"intent": "Review the query behind every metric that survived the cut list, correctness before performance, with corrected SQL and the queries that prove the fix.",
|
|
20
|
+
"skills": ["sql-review"],
|
|
21
|
+
"gate": {
|
|
22
|
+
"id": "G9",
|
|
23
|
+
"kind": "review",
|
|
24
|
+
"verdicts": ["READY", "NOT READY", "BLOCKED"],
|
|
25
|
+
"revise_to": "spec"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"id": "sign-off",
|
|
30
|
+
"type": "gate",
|
|
31
|
+
"intent": "Each metric's owner confirms its definition and the number its reviewed query now produces. The critique informs that decision; it does not make it.",
|
|
32
|
+
"skills": ["doc-critique"],
|
|
33
|
+
"gate": {
|
|
34
|
+
"id": "G9.1",
|
|
35
|
+
"kind": "human",
|
|
36
|
+
"verdicts": ["PROCEED", "PROCEED WITH CONDITIONS", "REVISE", "BLOCKED"],
|
|
37
|
+
"revise_to": "spec"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: comply
|
|
3
|
+
description: Take a change that carries risk (a launch, a new vendor, a new use of personal data) to a risk position its owner has signed: a risk register scored inherent and residual, a DPIA when personal data is processed, the evidence behind every control the ratings rely on, a script that checks the register, and a sign-off only the named owner can give. Use when the user says "get this ready for risk sign-off", "run the comply loop", "what do we need before we can launch this", or needs a register, a DPIA and audit evidence that agree with each other.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# comply
|
|
7
|
+
|
|
8
|
+
One change, from its risks to a signature. Each of the three documents is useful alone. The
|
|
9
|
+
loop exists for the places where they disagree. A register can rate a risk low because of a
|
|
10
|
+
control that leaves no record. A DPIA can find a high risk to people that never reaches the
|
|
11
|
+
register its owner reads. An evidence map can show a gap that nobody connects to the rating
|
|
12
|
+
that relied on it. The loop makes the three agree. Then the register's own script checks
|
|
13
|
+
them, and the person who owns the risk decides.
|
|
14
|
+
|
|
15
|
+
This loop prepares material for a qualified reviewer: a DPO, a risk committee, an auditor.
|
|
16
|
+
It is not legal or audit advice, and every document it produces says so once, at the top.
|
|
17
|
+
|
|
18
|
+
## Stages
|
|
19
|
+
|
|
20
|
+
| # | Stage | Type | Skills | Gate |
|
|
21
|
+
|---|---|---|---|---|
|
|
22
|
+
| 1 | `register` | generate | [`risk-register`](../../skills/risk-register/SKILL.md) | β |
|
|
23
|
+
| 2 | `assess` | generate | [`dpia`](../../skills/dpia/SKILL.md) | β |
|
|
24
|
+
| 3 | `evidence` | generate | [`soc2-evidence-map`](../../skills/soc2-evidence-map/SKILL.md) | β |
|
|
25
|
+
| 4 | `check` | verify | [`risk-register`](../../skills/risk-register/SKILL.md) | **G8** mechanical |
|
|
26
|
+
| 5 | `sign-off` | gate | [`doc-critique`](../../../core/skills/doc-critique/SKILL.md) | **G8.1** human |
|
|
27
|
+
|
|
28
|
+
## How to run this loop
|
|
29
|
+
|
|
30
|
+
1. **Register the change's risks.** Run `risk-register` with the change as its scope and the
|
|
31
|
+
objective it threatens. Save `risk-register.csv`. Owners come from the person; a missing
|
|
32
|
+
owner stays an error to resolve, never a guessed name.
|
|
33
|
+
2. **Screen and assess personal data.** Run `dpia` on the processing the change introduces.
|
|
34
|
+
- When the screening finds no criteria that apply, keep its short screening record and go on.
|
|
35
|
+
- When a DPIA is needed, complete it. Then give each risk to people still at medium or high
|
|
36
|
+
residual its own register row, with the harm to people as the consequence, the same
|
|
37
|
+
owner as the DPIA measure, and `DPIA` in its controls cell. The DPIA keeps scoring harm to
|
|
38
|
+
people; the register row is how its owner sees it.
|
|
39
|
+
3. **Evidence every control the ratings rely on.** List the controls named in the register's
|
|
40
|
+
residual ratings and the DPIA's in-place measures, and run `soc2-evidence-map` on that
|
|
41
|
+
list. Use the organisation's SOC 2 scope when it has one; otherwise use the skill's
|
|
42
|
+
default, Security only, tagged `[assumption]`.
|
|
43
|
+
4. **Put the evidence gaps back into the register.** A residual rating can count only
|
|
44
|
+
controls that operate today. For each control the map marks `gap`:
|
|
45
|
+
- set the residual likelihood and impact of every risk that relied on it back to their
|
|
46
|
+
inherent values;
|
|
47
|
+
- move the control to a dated action under the treatment.
|
|
48
|
+
|
|
49
|
+
A `partial` control keeps its credit only for what its evidence covers; say what that is
|
|
50
|
+
in the summary for the owner. Don't lower a rating to make a heat map look better.
|
|
51
|
+
5. **Check at G8.** Run the register's script on the final CSV:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
# Claude Code (the loop and risk-register installed side by side)
|
|
55
|
+
python3 "${CLAUDE_SKILL_DIR}/../risk-register/scripts/check_register.py" risk-register.csv --top 5 --out register-check.txt
|
|
56
|
+
# Other IDEs (from the folder the skills were installed into)
|
|
57
|
+
python3 risk-register/scripts/check_register.py risk-register.csv --top 5 --out register-check.txt
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The exit code is the verdict: `0` is `READY`, `1` is `NOT READY`, `2` is `BLOCKED`.
|
|
61
|
+
`NOT READY` returns to `register` to fix what the script names, then runs `check` again.
|
|
62
|
+
Report its warnings to the person; don't edit scores to silence them.
|
|
63
|
+
6. **Critique, then put the decision to the owner at G8.1.** Run `doc-critique` on the
|
|
64
|
+
summary the owner will read. Then show them:
|
|
65
|
+
- the residual heat map from the script, as printed;
|
|
66
|
+
- every risk they are asked to accept;
|
|
67
|
+
- the evidence gaps and their dates;
|
|
68
|
+
- the DPIA's view on prior consultation.
|
|
69
|
+
|
|
70
|
+
Each owner accepts their own risks. A high or critical residual risk that its own owner
|
|
71
|
+
accepts needs a sign-off from someone above them, named in the summary.
|
|
72
|
+
|
|
73
|
+
## Verdicts
|
|
74
|
+
|
|
75
|
+
Defined in [`contracts/terminals.json`](../../../../contracts/terminals.json).
|
|
76
|
+
|
|
77
|
+
- G8 emits `READY`, `NOT READY` or `BLOCKED`, from the script's exit code alone.
|
|
78
|
+
- G8.1 emits:
|
|
79
|
+
- `PROCEED`: every residual risk is accepted or treated, by its named owner.
|
|
80
|
+
- `PROCEED WITH CONDITIONS`: accepted, subject to named actions with dates, each recorded
|
|
81
|
+
as a control and `due_date` on its risk.
|
|
82
|
+
- `REVISE`: the owner rejects a rating or a treatment. Return to `register` with their reasons.
|
|
83
|
+
- `RECONSIDER`: the change shouldn't go ahead as designed, for example when a residual risk
|
|
84
|
+
to people stays high and the design has to change before prior consultation. The loop
|
|
85
|
+
ends here.
|
|
86
|
+
- `BLOCKED`: no owner is available, or the DPO's advice is required and not yet given.
|
|
87
|
+
|
|
88
|
+
## Non-interactive runs
|
|
89
|
+
|
|
90
|
+
With no person present, G8.1 cannot be satisfied. Run every stage through G8, deliver the
|
|
91
|
+
documents, and emit `BLOCKED: need sign-off from <the named owners>`. Never accept a risk
|
|
92
|
+
on someone's behalf, never fill an owner, a retention period or a vendor location with a
|
|
93
|
+
plausible guess, and leave the DPIA's DPO advice and decision fields blank.
|
|
94
|
+
|
|
95
|
+
## When a stage's skill is not installed
|
|
96
|
+
|
|
97
|
+
Name the missing skill and skilldrop as its source, do the minimal inline version of that
|
|
98
|
+
stage, and record that it ran degraded. Without `risk-register`, G8 has no script. Say so,
|
|
99
|
+
check the register by hand against the script's rules (every required field, owners that
|
|
100
|
+
name a person, residual never above inherent, dates that are dates), and mark G8 degraded.
|
|
101
|
+
Never skip G8.1.
|
|
102
|
+
|
|
103
|
+
## Run log (opt-in)
|
|
104
|
+
|
|
105
|
+
Only when the environment variable `SKILLDROP_LOOP_LOG` is set to a file path: after each
|
|
106
|
+
gate's verdict, append one JSON line to that file. Nothing else is recorded, and nothing is
|
|
107
|
+
sent anywhere. When the variable isn't set, or you can't read it, skip this section.
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
printf '%s\n' '{"ts":"2026-10-19T14:03:00Z","loop":"comply","stage":"check","gate":"G8","verdict":"READY","round":1}' >> "$SKILLDROP_LOOP_LOG"
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Fields: `ts` (UTC), `loop`, `stage`, `gate` (`null` for a stage without one), `verdict` (the
|
|
114
|
+
gate's exact word), and `round` (1 for the first attempt, counting returns under the cap).
|
|
115
|
+
The log holds verdicts only: never a risk, an owner's name or any personal data.
|
|
116
|
+
`skilldrop loop-stats` summarises the file.
|
|
117
|
+
|
|
118
|
+
## Quality bar
|
|
119
|
+
|
|
120
|
+
- **The three documents agree.** Every DPIA risk at medium or high residual has a register row, and every control a residual rating relies on appears in the evidence map.
|
|
121
|
+
- **A residual rating counts only evidenced controls.** A control with an evidence gap gives no credit until the evidence exists, and the gap has a dated action.
|
|
122
|
+
- **G8 is the script's exit code.** The register reaches its owner only with zero errors.
|
|
123
|
+
- **Only a named owner accepts a risk.** High and critical residual risks accepted by their own owner carry a second, named sign-off.
|
|
124
|
+
- **Not legal or audit advice, said once.** The DPO's advice, the decision and the sign-off fields are left for the people who give them.
|
|
125
|
+
|
|
126
|
+
## Anti-patterns to avoid
|
|
127
|
+
|
|
128
|
+
- β **Filing the DPIA beside the register instead of into it.** A high risk to people that isn't on the owner's list doesn't get signed off; it gets missed.
|
|
129
|
+
- β **Leaving a residual rating on a control the evidence map marked `gap`.** That is a planned control scored as a real one, the exact thing the register bans.
|
|
130
|
+
- β **Editing the register to turn G8 green.** Fix the risk the script names; don't delete the row or soften the score.
|
|
131
|
+
- β **Running the evidence map on the whole SOC 2 scope.** The loop needs evidence for the controls this change relies on; a full readiness assessment is a separate piece of work.
|
|
132
|
+
- β **Self-signing at G8.1.** A clean check says the register is well-formed. It cannot say the risk is acceptable.
|
|
133
|
+
- β **Writing a guessed vendor location or retention period** to complete the DPIA. It stays `[to confirm: β¦]` and becomes a DPO question.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"query": "get the new payroll vendor ready for risk sign-off", "should_trigger": true},
|
|
3
|
+
{"query": "run the comply loop on our launch of in-store facial recognition", "should_trigger": true},
|
|
4
|
+
{"query": "what do we need before we can launch this feature that profiles customers?", "should_trigger": true},
|
|
5
|
+
{"query": "I need a risk register, a DPIA and the audit evidence for this change, and they have to agree", "should_trigger": true},
|
|
6
|
+
{"query": "score these five risks on a 5x5", "should_trigger": false},
|
|
7
|
+
{"query": "do we need a DPIA for the new CRM?", "should_trigger": false},
|
|
8
|
+
{"query": "fill in this customer's security questionnaire", "should_trigger": false}
|
|
9
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"loop_name": "comply",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": "evidence-gap-reverts-the-rating",
|
|
6
|
+
"prompt": "Run the comply loop. We're moving payroll to a new SaaS vendor, Paywell, on 1 December. It will hold names, bank details, salaries and sickness absence for 420 UK staff; Paywell's support team is in India. The HR director, Priya Shah, owns the change. Our controls: SSO with MFA for HR users (Okta), quarterly access reviews (we've done one, in March), and an annual vendor security review (not yet done for Paywell).",
|
|
7
|
+
"assertions": [
|
|
8
|
+
"Writes each risk as cause, event and consequence with a named owner, inherent and residual scores on a stated scale",
|
|
9
|
+
"Finds that a DPIA is needed (special category health data, transfer outside the UK) and completes it, leaving the DPO advice and decision blank",
|
|
10
|
+
"Adds a register row for each DPIA risk to people left at medium or high residual",
|
|
11
|
+
"Marks the quarterly access review as partial or a gap for a Type II period (one review since March) and the Paywell vendor review as a gap",
|
|
12
|
+
"Sets the residual rating of any risk relying on the missing vendor review back to its inherent rating",
|
|
13
|
+
"Runs check_register.py and reports G8 from its exit code",
|
|
14
|
+
"Stops at G8.1 with BLOCKED or a request for Priya Shah's decision; does not accept any risk on the owner's behalf",
|
|
15
|
+
"Does not invent Paywell's data centre location or a retention period; marks them [to confirm: ...]"
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "script-failure-returns-to-register",
|
|
20
|
+
"prompt": "Continuing the comply loop. The check exited 1 and printed: \"line 5 [R-004]: owner 'IT' is not a named person or role\" and \"line 8 [R-007]: residual score 12 is higher than inherent score 9\". Fix it.",
|
|
21
|
+
"assertions": [
|
|
22
|
+
"Treats the output as G8 NOT READY and returns to the register stage",
|
|
23
|
+
"Asks for or names a person or role to own R-004 rather than inventing one",
|
|
24
|
+
"Corrects R-007 by revisiting its scores or controls, not by deleting the row",
|
|
25
|
+
"Re-runs the script and reports the new exit code before going to sign-off",
|
|
26
|
+
"Counts this as round 2 under the cap of 3"
|
|
27
|
+
]
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "RFC-0052. A change to a risk position its owner has signed. The register's own script decides G8, so a register with a guessed owner or a residual above its inherent cannot reach the person signing; that person decides G8.1, because accepting a risk is theirs alone.",
|
|
3
|
+
"name": "comply",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"description": "Take a change that carries risk (a launch, a new vendor, a new use of personal data) to a risk position its owner has signed: a risk register scored inherent and residual, a DPIA when personal data is processed, the evidence behind every control the ratings rely on, a script that checks the register, and a sign-off only the named owner can give. Use when the user says \"get this ready for risk sign-off\", \"run the comply loop\", \"what do we need before we can launch this\", or needs a register, a DPIA and audit evidence that agree with each other.",
|
|
6
|
+
"entrypoint": "LOOP.md",
|
|
7
|
+
"kind": "loop",
|
|
8
|
+
"cap": 3,
|
|
9
|
+
"stages": [
|
|
10
|
+
{
|
|
11
|
+
"id": "register",
|
|
12
|
+
"type": "generate",
|
|
13
|
+
"intent": "Write the change's risks as cause, event and consequence, scored inherent and residual on a stated scale, each with a named owner and a treatment.",
|
|
14
|
+
"skills": ["risk-register"]
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "assess",
|
|
18
|
+
"type": "generate",
|
|
19
|
+
"intent": "Screen for a DPIA and, where one is needed, score the harm to people; each risk to people left at medium or high gets its own row in the register.",
|
|
20
|
+
"skills": ["dpia"]
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"id": "evidence",
|
|
24
|
+
"type": "generate",
|
|
25
|
+
"intent": "For every control the residual ratings and the DPIA's measures rely on, name the evidence that it operates today, and mark the gaps.",
|
|
26
|
+
"skills": ["soc2-evidence-map"]
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"id": "check",
|
|
30
|
+
"type": "verify",
|
|
31
|
+
"intent": "Run the register's own check, after evidence gaps have put residual ratings back where the evidence leaves them.",
|
|
32
|
+
"skills": ["risk-register"],
|
|
33
|
+
"gate": {
|
|
34
|
+
"id": "G8",
|
|
35
|
+
"kind": "mechanical",
|
|
36
|
+
"script": "packs/grc/skills/risk-register/scripts/check_register.py",
|
|
37
|
+
"verdicts": ["READY", "NOT READY", "BLOCKED"],
|
|
38
|
+
"revise_to": "register"
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"id": "sign-off",
|
|
43
|
+
"type": "gate",
|
|
44
|
+
"intent": "The named owner accepts each residual risk, or doesn't. The critique informs that decision; it does not make it.",
|
|
45
|
+
"skills": ["doc-critique"],
|
|
46
|
+
"gate": {
|
|
47
|
+
"id": "G8.1",
|
|
48
|
+
"kind": "human",
|
|
49
|
+
"verdicts": ["PROCEED", "PROCEED WITH CONDITIONS", "REVISE", "RECONSIDER", "BLOCKED"],
|
|
50
|
+
"revise_to": "register"
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
]
|
|
54
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: research
|
|
3
|
+
description: Take a question that turns on why something happened to an answer the evidence can carry: a plan tied to the decision it serves, a synthesis of the sources with every claim cited, and rival explanations ranked by how little evidence contradicts them, with a gate that sends the work back for the observation that would tell the leaders apart. Use when the user says "research whyβ¦", "find out what's causingβ¦", "run the research loop", or needs a decision-ready answer on which explanation the evidence supports.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# research
|
|
7
|
+
|
|
8
|
+
One question, from scoping to an answer someone can decide on. The three skills each work
|
|
9
|
+
alone; in order, each one constrains the next. The plan fixes what would change the decision
|
|
10
|
+
before anyone reads a source, so the findings can't be read to fit. The synthesis cites every
|
|
11
|
+
claim, so the comparison rates only evidence that exists. The gate then asks whether that
|
|
12
|
+
evidence separates the explanations, which none of the three can judge about its own output.
|
|
13
|
+
|
|
14
|
+
## Stages
|
|
15
|
+
|
|
16
|
+
| # | Stage | Type | Skills | Gate |
|
|
17
|
+
|---|---|---|---|---|
|
|
18
|
+
| 1 | `plan` | generate | [`research-plan`](../../skills/research-plan/SKILL.md) | β |
|
|
19
|
+
| 2 | `synthesise` | generate | [`source-synthesis`](../../skills/source-synthesis/SKILL.md) | β |
|
|
20
|
+
| 3 | `compare` | verify | [`hypothesis-comparison`](../../skills/hypothesis-comparison/SKILL.md) | **G7** review |
|
|
21
|
+
|
|
22
|
+
## How to run this loop
|
|
23
|
+
|
|
24
|
+
1. **Plan before reading.** Run `research-plan` on the question. It asks what decision the
|
|
25
|
+
research serves, who makes it and by when; don't skip that. Save the plan as `plan.md`.
|
|
26
|
+
Carry three things from it into every later stage: the decision, the findings that would
|
|
27
|
+
change it, and the stopping rule.
|
|
28
|
+
2. **Gather the sources the plan names.** The person supplies them, or you collect them under
|
|
29
|
+
the plan's inclusion and exclusion criteria with the search strings it wrote. A source the
|
|
30
|
+
criteria exclude stays out, however convenient. List each as `S1: β¦` in `sources.md`.
|
|
31
|
+
3. **Synthesise, cited.** Run `source-synthesis` in its cited form (an evidence summary with
|
|
32
|
+
`[S1:p4]` citations), on the plan's question. It counts a source that repeats another once,
|
|
33
|
+
and its citation check runs before you hand anything on. Save it as `synthesis.md`.
|
|
34
|
+
4. **Compare the explanations.** Run `hypothesis-comparison` with the observation from the
|
|
35
|
+
plan as the thing to explain and the synthesis's findings as the evidence, keeping their
|
|
36
|
+
citations. Every evidence row cites a source from `sources.md` or is marked
|
|
37
|
+
`[assumption]`. The comparison names the leader, the diagnostic evidence, the linchpins and
|
|
38
|
+
the next observation that would tell the top two apart.
|
|
39
|
+
5. **Judge the evidence at G7.** Ask four questions, then choose the verdict below:
|
|
40
|
+
- Does at least one diagnostic row separate the leader from the runner-up?
|
|
41
|
+
- Does any linchpin rest on a single source or an `[assumption]`?
|
|
42
|
+
- Is the plan's stopping rule met?
|
|
43
|
+
- Would the finding move the decision past a threshold the plan set?
|
|
44
|
+
6. **On `REVISE`, get the observation, not more of the same.** Return to `synthesise` with the
|
|
45
|
+
one observation the comparison named as discriminating. Add it to `sources.md` as a new
|
|
46
|
+
source, re-synthesise, and compare again. More sources that the top two both predict
|
|
47
|
+
equally add nothing. Each return counts against the cap. When only a person can get the
|
|
48
|
+
observation (a data pull, an interview, a log nobody gave you), stop here: say what to
|
|
49
|
+
get, from whom, and what each result would mean, and resume when they add it.
|
|
50
|
+
7. **Hand over the answer, then the verdict.** Lead with the explanation the evidence favours
|
|
51
|
+
and its confidence, then what that means for the decision named in the plan, then the
|
|
52
|
+
open conditions. The matrix and the synthesis go underneath, for anyone who wants to
|
|
53
|
+
check. Close with the gate on its own line, `G7: <VERDICT>`, and one sentence of reasons.
|
|
54
|
+
The skills keep their own terms out of a reply; the loop's verdict is the exception,
|
|
55
|
+
because the run log and the next round both read it.
|
|
56
|
+
|
|
57
|
+
## Verdicts
|
|
58
|
+
|
|
59
|
+
Defined in [`contracts/terminals.json`](../../../../contracts/terminals.json). G7 emits:
|
|
60
|
+
|
|
61
|
+
- `PROCEED`: diagnostic evidence separates the leader, no linchpin rests on one source or an
|
|
62
|
+
assumption, and the stopping rule is met.
|
|
63
|
+
- `PROCEED WITH CONDITIONS`: the leader is clear, but a named linchpin is unchecked or a
|
|
64
|
+
sub-question is still open. The conditions travel with the answer: each one says what to
|
|
65
|
+
check and how the decision changes if it comes out the other way.
|
|
66
|
+
- `REVISE`: the top two are not separated by any diagnostic row, or the leader rests on one
|
|
67
|
+
weak row. Return to `synthesise` with the discriminating observation.
|
|
68
|
+
- `RECONSIDER`: every hypothesis is contradicted by strong evidence, or the observation
|
|
69
|
+
itself turned out wrong (a tracking change, a definition change). The question needs
|
|
70
|
+
re-planning, so the loop ends here and a new run starts at `plan`.
|
|
71
|
+
- `BLOCKED`: no decision to serve, no observation to explain, or no sources.
|
|
72
|
+
|
|
73
|
+
On the cap-th `REVISE`, emit `PROCEED WITH CONDITIONS` with the leaders still tied and say
|
|
74
|
+
plainly that the evidence doesn't settle it: the decision will be made under that
|
|
75
|
+
uncertainty, and the discriminating observation is the first thing to get.
|
|
76
|
+
|
|
77
|
+
## Non-interactive runs
|
|
78
|
+
|
|
79
|
+
With no person to name the decision, `research-plan` either infers it as an `[assumption]` or
|
|
80
|
+
emits `BLOCKED: need the decision this research informs, who makes it, and by when`. Carry
|
|
81
|
+
that `BLOCKED` through as the loop's verdict. Run no further stages on an invented decision:
|
|
82
|
+
a stopping rule set against a guessed decision stops at the wrong place. With no sources and
|
|
83
|
+
nothing to collect them from, emit `BLOCKED: need the sources, or access to collect them
|
|
84
|
+
under the plan's criteria`.
|
|
85
|
+
|
|
86
|
+
## When a stage's skill is not installed
|
|
87
|
+
|
|
88
|
+
Name the missing skill and skilldrop as its source, do the minimal inline version of that
|
|
89
|
+
stage, and record that it ran degraded. A degraded `compare` still builds a matrix that
|
|
90
|
+
ranks by inconsistency and names its linchpins, because G7 can't be judged without them.
|
|
91
|
+
Never skip G7.
|
|
92
|
+
|
|
93
|
+
## Run log (opt-in)
|
|
94
|
+
|
|
95
|
+
Only when the environment variable `SKILLDROP_LOOP_LOG` is set to a file path: after each
|
|
96
|
+
gate's verdict, append one JSON line to that file. Nothing else is recorded, and nothing is
|
|
97
|
+
sent anywhere. When the variable isn't set, or you can't read it, skip this section.
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
printf '%s\n' '{"ts":"2026-10-19T14:03:00Z","loop":"research","stage":"compare","gate":"G7","verdict":"REVISE","round":1}' >> "$SKILLDROP_LOOP_LOG"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Fields: `ts` (UTC), `loop`, `stage`, `gate` (`null` for a stage without one), `verdict` (the
|
|
104
|
+
gate's exact word), and `round` (1 for the first attempt, counting returns under the cap).
|
|
105
|
+
The log holds verdicts only: never the question, the sources or the findings.
|
|
106
|
+
`skilldrop loop-stats` summarises the file.
|
|
107
|
+
|
|
108
|
+
## Quality bar
|
|
109
|
+
|
|
110
|
+
- **What would change the decision is written before any source is read.** Thresholds set after the findings are read get set to fit them.
|
|
111
|
+
- **Every evidence row in the matrix cites a source in `sources.md` or is marked `[assumption]`.** A matrix of uncited evidence ranks a favourite theory.
|
|
112
|
+
- **G7 is judged on diagnostic evidence.** Rows that every hypothesis predicts equally are named and set aside, however many there are.
|
|
113
|
+
- **A `REVISE` brings back the one observation that discriminates.** It does not bring back more sources of the same kind.
|
|
114
|
+
- **The answer leads with the explanation and what it means for the decision.** The matrix is the working, not the reply.
|
|
115
|
+
|
|
116
|
+
## Anti-patterns to avoid
|
|
117
|
+
|
|
118
|
+
- β **Synthesising before the plan exists.** The sources then decide the question, and the stopping rule is whenever the reading runs out.
|
|
119
|
+
- β **Rating evidence the synthesis didn't cite.** An observation from memory or a plausible industry figure is an `[assumption]` row, weighted as one.
|
|
120
|
+
- β **Counting a press release and the three articles that quote it as four sources.** The synthesis counts them once, and so does the matrix.
|
|
121
|
+
- β **Passing G7 on volume.** Ten consistent rows that every rival also explains don't separate anything.
|
|
122
|
+
- β **Looping past the cap in search of certainty.** At the cap, say that the evidence doesn't settle it and name the next check.
|
|
123
|
+
- β **Re-planning inside the loop after `RECONSIDER`.** A new question is a new run, with its own plan and stopping rule.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"query": "research why our trial-to-paid conversion dropped in September", "should_trigger": true},
|
|
3
|
+
{"query": "run the research loop on what's causing the rise in support tickets", "should_trigger": true},
|
|
4
|
+
{"query": "find out which explanation the evidence supports for the churn spike, so I can decide what to fix", "should_trigger": true},
|
|
5
|
+
{"query": "I need a decision-ready answer on why the pilot underperformed", "should_trigger": true},
|
|
6
|
+
{"query": "summarise these three papers for me", "should_trigger": false},
|
|
7
|
+
{"query": "help me scope a research plan for entering the German market", "should_trigger": false},
|
|
8
|
+
{"query": "build an ACH matrix from this evidence list", "should_trigger": false}
|
|
9
|
+
]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"loop_name": "research",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": "plans-before-reading",
|
|
6
|
+
"prompt": "Run the research loop: why did our trial-to-paid conversion drop from 14% to 9% in September? I've put the sources in sources/ (a product analytics export, the September release notes, 12 sales call notes, and a competitor's pricing page). The growth lead decides by 1 November whether to roll back the new onboarding flow.",
|
|
7
|
+
"assertions": [
|
|
8
|
+
"Runs research-plan first and writes, before synthesising, what finding would change the roll-back decision",
|
|
9
|
+
"States the stopping rule before reading the sources",
|
|
10
|
+
"The synthesis cites every claim to a listed source and location",
|
|
11
|
+
"The comparison includes a measurement hypothesis (tracking or definition change) alongside the onboarding flow and the competitor's pricing",
|
|
12
|
+
"Every evidence row in the matrix cites a listed source or is marked [assumption]",
|
|
13
|
+
"Ends with a G7 verdict from PROCEED, PROCEED WITH CONDITIONS, REVISE, RECONSIDER or BLOCKED, and the reasons for it",
|
|
14
|
+
"The reply leads with the explanation the evidence favours and what it means for the roll-back, not with the matrix"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"id": "revise-brings-the-discriminating-observation",
|
|
19
|
+
"prompt": "Continuing the research loop. The comparison left H1 (the new onboarding flow) and H3 (the competitor's price cut on 4 September) tied, and named as discriminating: conversion for trials that started before the onboarding release on 11 September. I've added that cohort export as sources/cohorts.csv.",
|
|
20
|
+
"assertions": [
|
|
21
|
+
"Treats the tie as G7 REVISE and returns to synthesise, not to plan",
|
|
22
|
+
"Adds the cohort export as a new numbered source and re-synthesises with it cited",
|
|
23
|
+
"Re-runs the comparison with the new evidence row rated across every hypothesis",
|
|
24
|
+
"Counts this as round 2 under the cap of 3",
|
|
25
|
+
"Does not add further sources that both H1 and H3 predict equally"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"id": "blocked-without-a-decision",
|
|
30
|
+
"prompt": "Run the research loop on why remote work is good. Non-interactive, no follow-up questions.",
|
|
31
|
+
"assertions": [
|
|
32
|
+
"Emits BLOCKED: need the decision this research informs, who makes it, and by when",
|
|
33
|
+
"Does not run source-synthesis or hypothesis-comparison",
|
|
34
|
+
"Does not invent a decision or gather sources",
|
|
35
|
+
"Says in plain words what it will produce once it has the decision"
|
|
36
|
+
]
|
|
37
|
+
}
|
|
38
|
+
]
|
|
39
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "RFC-0052. A question to an answer the evidence can carry. The gate sits after the comparison because a synthesis can't tell you whether its own evidence separates the explanations; the matrix can.",
|
|
3
|
+
"name": "research",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"description": "Take a question that turns on why something happened to an answer the evidence can carry: a plan tied to the decision it serves, a synthesis of the sources with every claim cited, and rival explanations ranked by how little evidence contradicts them, with a gate that sends the work back for the observation that would tell the leaders apart. Use when the user says \"research whyβ¦\", \"find out what's causingβ¦\", \"run the research loop\", or needs a decision-ready answer on which explanation the evidence supports.",
|
|
6
|
+
"entrypoint": "LOOP.md",
|
|
7
|
+
"kind": "loop",
|
|
8
|
+
"cap": 3,
|
|
9
|
+
"stages": [
|
|
10
|
+
{
|
|
11
|
+
"id": "plan",
|
|
12
|
+
"type": "generate",
|
|
13
|
+
"intent": "Pin the decision the research serves, the precise question, what finding would change the decision, and when to stop, before any source is read.",
|
|
14
|
+
"skills": ["research-plan"]
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "synthesise",
|
|
18
|
+
"type": "generate",
|
|
19
|
+
"intent": "Say what the sources gathered under the plan support, every claim cited to a location, a repeated source counted once.",
|
|
20
|
+
"skills": ["source-synthesis"]
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"id": "compare",
|
|
24
|
+
"type": "verify",
|
|
25
|
+
"intent": "Rank the rival explanations against the cited evidence, then judge whether that evidence is strong enough to carry the decision.",
|
|
26
|
+
"skills": ["hypothesis-comparison"],
|
|
27
|
+
"gate": {
|
|
28
|
+
"id": "G7",
|
|
29
|
+
"kind": "review",
|
|
30
|
+
"verdicts": ["PROCEED", "PROCEED WITH CONDITIONS", "REVISE", "RECONSIDER", "BLOCKED"],
|
|
31
|
+
"revise_to": "synthesise"
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|