role-os 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/bin/roleos.mjs +10 -0
- package/package.json +1 -1
- package/src/citation-panel.mjs +9 -7
- package/src/specialist/budget-consult.mjs +120 -0
- package/src/specialist/client.mjs +131 -0
- package/src/specialist/dispatch.mjs +237 -0
- package/src/specialist/events.mjs +56 -0
- package/src/specialist/gate.mjs +202 -0
- package/src/specialist/registry.mjs +219 -0
- package/src/specialist/shadow.mjs +122 -0
- package/src/specialist/state.mjs +125 -0
- package/src/specialist-cmd.mjs +378 -0
- package/src/verify-citations.mjs +1 -0
- package/starter-pack/policy/specialist-tier.md +288 -0
- package/starter-pack/schemas/specialist.md +155 -0
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `roleos specialist <subcommand>` — the operator surface for the specialist tier.
|
|
3
|
+
*
|
|
4
|
+
* Subcommands:
|
|
5
|
+
* list — show all roles in the registry + active version + cert level
|
|
6
|
+
* status [--role <role>] — registry + halt state + quota state, per role
|
|
7
|
+
* register <role> <version.json> — add a new version to a role's versions[] (or create role)
|
|
8
|
+
* promote <role> <version-id> — set active_version (refused if version is L0 — Reject 2)
|
|
9
|
+
* rollback <role> <version-id> — set active_version to a prior certified version (the
|
|
10
|
+
* NAMED COMPENSATOR; pure pointer swap, no retrain)
|
|
11
|
+
* clear-halt <role> [--reason] — clear a shadow-probe halt
|
|
12
|
+
*
|
|
13
|
+
* All mutating subcommands write a receipt to the events log.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
17
|
+
import { resolve } from "node:path";
|
|
18
|
+
import {
|
|
19
|
+
loadRegistry,
|
|
20
|
+
saveRegistry,
|
|
21
|
+
emptyRegistry,
|
|
22
|
+
resolveActiveVersion,
|
|
23
|
+
} from "./specialist/registry.mjs";
|
|
24
|
+
import {
|
|
25
|
+
loadState,
|
|
26
|
+
saveState,
|
|
27
|
+
emptyState,
|
|
28
|
+
getHalt,
|
|
29
|
+
setHalt,
|
|
30
|
+
quotaStateFor,
|
|
31
|
+
} from "./specialist/state.mjs";
|
|
32
|
+
import {
|
|
33
|
+
appendEvent,
|
|
34
|
+
readEvents,
|
|
35
|
+
} from "./specialist/events.mjs";
|
|
36
|
+
import { appendClearHaltEvent } from "./specialist/shadow.mjs";
|
|
37
|
+
|
|
38
|
+
const DEFAULT_REGISTRY_PATH = ".role-os/specialists.json";
|
|
39
|
+
const DEFAULT_STATE_PATH = ".role-os/specialist-state.json";
|
|
40
|
+
const DEFAULT_EVENTS_PATH = ".role-os/specialist-events.jsonl";
|
|
41
|
+
|
|
42
|
+
export async function specialistCommand(args) {
|
|
43
|
+
const sub = args[0];
|
|
44
|
+
const rest = args.slice(1);
|
|
45
|
+
switch (sub) {
|
|
46
|
+
case "list": return listSpecialists(rest);
|
|
47
|
+
case "status": return statusSpecialists(rest);
|
|
48
|
+
case "register": return registerSpecialist(rest);
|
|
49
|
+
case "promote": return promoteSpecialist(rest);
|
|
50
|
+
case "rollback": return rollbackSpecialist(rest);
|
|
51
|
+
case "clear-halt": return clearHaltSpecialist(rest);
|
|
52
|
+
case undefined:
|
|
53
|
+
case "--help":
|
|
54
|
+
case "-h":
|
|
55
|
+
case "help":
|
|
56
|
+
return printHelp();
|
|
57
|
+
default: {
|
|
58
|
+
const err = new Error(`Unknown specialist subcommand: ${sub}`);
|
|
59
|
+
err.exitCode = 1;
|
|
60
|
+
err.hint = "Run 'roleos specialist help' for usage.";
|
|
61
|
+
throw err;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function printHelp() {
|
|
67
|
+
console.log(`
|
|
68
|
+
roleos specialist — manage the specialist tier (trained adapters fronted by a fail-open gate).
|
|
69
|
+
|
|
70
|
+
Usage:
|
|
71
|
+
roleos specialist list
|
|
72
|
+
roleos specialist status [--role <role>] [--json]
|
|
73
|
+
roleos specialist register <role> <version.json>
|
|
74
|
+
roleos specialist promote <role> <version-id> [--operator <name>]
|
|
75
|
+
roleos specialist rollback <role> <version-id> [--operator <name>] [--reason <text>]
|
|
76
|
+
roleos specialist clear-halt <role> [--operator <name>] [--reason <text>]
|
|
77
|
+
|
|
78
|
+
Paths (overridable via env):
|
|
79
|
+
registry: ROLEOS_SPECIALISTS_PATH (default ${DEFAULT_REGISTRY_PATH})
|
|
80
|
+
state: ROLEOS_SPECIALIST_STATE_PATH (default ${DEFAULT_STATE_PATH})
|
|
81
|
+
events: ROLEOS_SPECIALIST_EVENTS_PATH (default ${DEFAULT_EVENTS_PATH})
|
|
82
|
+
|
|
83
|
+
Notes:
|
|
84
|
+
- 'promote' is refused if the target version is L0 (uncertified) — Reject 2.
|
|
85
|
+
- 'rollback' is the named compensator: a pure pointer swap, no retrain, no adapter delete.
|
|
86
|
+
- All mutating subcommands append an event to the events log.
|
|
87
|
+
- See starter-pack/policy/specialist-tier.md for the dispatch law and reject conditions.
|
|
88
|
+
`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ── list ────────────────────────────────────────────────────────────────────────────────────
|
|
92
|
+
|
|
93
|
+
function listSpecialists(args) {
|
|
94
|
+
const { flags } = parseArgs(args);
|
|
95
|
+
const registryPath = flags.registry ? resolve(flags.registry) : registryPathFromEnv();
|
|
96
|
+
const { registry, byRole, errors } = loadRegistry(registryPath);
|
|
97
|
+
if (errors.length) {
|
|
98
|
+
console.error("Registry errors:");
|
|
99
|
+
for (const e of errors) console.error(` - ${e}`);
|
|
100
|
+
process.exit(2);
|
|
101
|
+
}
|
|
102
|
+
if (registry.specialists.length === 0) {
|
|
103
|
+
console.log(`(no specialists registered at ${registryPath})`);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
console.log(`Registry: ${registryPath}`);
|
|
107
|
+
console.log(``);
|
|
108
|
+
for (const entry of registry.specialists) {
|
|
109
|
+
const active = entry.active_version
|
|
110
|
+
? entry.versions.find((v) => v.id === entry.active_version)
|
|
111
|
+
: null;
|
|
112
|
+
const activeStr = active
|
|
113
|
+
? `active=${active.id} (${active.certified_level})`
|
|
114
|
+
: `active=null (no certified specialist routed for this role)`;
|
|
115
|
+
console.log(` ${entry.role} — ${activeStr}, quota=${entry.workload_quota}, versions=${entry.versions.length}`);
|
|
116
|
+
for (const v of entry.versions) {
|
|
117
|
+
const tag = v.id === entry.active_version ? "*" : " ";
|
|
118
|
+
console.log(` ${tag} ${v.id} ${v.certified_level} base=${v.base_model} adapter=${v.adapter_id}`);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ── status ──────────────────────────────────────────────────────────────────────────────────
|
|
124
|
+
|
|
125
|
+
function statusSpecialists(args) {
|
|
126
|
+
const { flags } = parseArgs(args);
|
|
127
|
+
const registryPath = flags.registry ? resolve(flags.registry) : registryPathFromEnv();
|
|
128
|
+
const statePath = flags.state ? resolve(flags.state) : statePathFromEnv();
|
|
129
|
+
const eventsPath = flags.events ? resolve(flags.events) : eventsPathFromEnv();
|
|
130
|
+
const windowSize = Number.isFinite(Number(flags.window)) ? Number(flags.window) : 200;
|
|
131
|
+
|
|
132
|
+
const { byRole, errors } = loadRegistry(registryPath);
|
|
133
|
+
if (errors.length) {
|
|
134
|
+
console.error("Registry errors:");
|
|
135
|
+
for (const e of errors) console.error(` - ${e}`);
|
|
136
|
+
process.exit(2);
|
|
137
|
+
}
|
|
138
|
+
let state;
|
|
139
|
+
try { state = loadState(statePath); }
|
|
140
|
+
catch (err) {
|
|
141
|
+
console.error(`State load error: ${err.message}`);
|
|
142
|
+
state = emptyState();
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const roles = flags.role ? [flags.role] : [...byRole.keys()];
|
|
146
|
+
const report = roles.map((role) => {
|
|
147
|
+
const entry = byRole.get(role);
|
|
148
|
+
if (!entry) return { role, error: "no_registry_entry" };
|
|
149
|
+
const active = entry.active_version
|
|
150
|
+
? entry.versions.find((v) => v.id === entry.active_version)
|
|
151
|
+
: null;
|
|
152
|
+
const quota = quotaStateFor(state, role, windowSize);
|
|
153
|
+
const halt = getHalt(state, role);
|
|
154
|
+
return {
|
|
155
|
+
role,
|
|
156
|
+
backend_url: entry.backend_url,
|
|
157
|
+
active_version: entry.active_version,
|
|
158
|
+
certified_level: active ? active.certified_level : null,
|
|
159
|
+
quota: { used: quota.used, window: quota.window, share: quota.used / quota.window, cap: entry.workload_quota },
|
|
160
|
+
halt: halt.halted ? { halted: true, reason: halt.reason, since: halt.since } : { halted: false },
|
|
161
|
+
};
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
if (flags.json === true) {
|
|
165
|
+
console.log(JSON.stringify({ registry: registryPath, state: statePath, events: eventsPath, roles: report }, null, 2));
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
console.log(`Registry: ${registryPath}`);
|
|
169
|
+
console.log(`State: ${statePath}`);
|
|
170
|
+
console.log(`Events: ${eventsPath}`);
|
|
171
|
+
console.log(``);
|
|
172
|
+
for (const r of report) {
|
|
173
|
+
if (r.error) {
|
|
174
|
+
console.log(` ${r.role}: ERROR ${r.error}`);
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
const haltStr = r.halt.halted ? `HALTED — ${r.halt.reason}` : "ok";
|
|
178
|
+
console.log(` ${r.role}:`);
|
|
179
|
+
console.log(` active: ${r.active_version || "(none)"} (${r.certified_level || "-"})`);
|
|
180
|
+
console.log(` quota: ${r.quota.used}/${r.quota.window} (share=${(r.quota.share * 100).toFixed(1)}%, cap=${(r.quota.cap * 100).toFixed(0)}%)`);
|
|
181
|
+
console.log(` backend: ${r.backend_url}`);
|
|
182
|
+
console.log(` halt: ${haltStr}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ── register ────────────────────────────────────────────────────────────────────────────────
|
|
187
|
+
|
|
188
|
+
function registerSpecialist(args) {
|
|
189
|
+
const { flags, positional } = parseArgs(args);
|
|
190
|
+
const role = positional[0];
|
|
191
|
+
const versionFile = positional[1];
|
|
192
|
+
if (!role || !versionFile) {
|
|
193
|
+
throwUsage("register <role> <version.json>");
|
|
194
|
+
}
|
|
195
|
+
if (!existsSync(versionFile)) {
|
|
196
|
+
throwUsage(`version file not found: ${versionFile}`);
|
|
197
|
+
}
|
|
198
|
+
let version;
|
|
199
|
+
try { version = JSON.parse(readFileSync(versionFile, "utf8")); }
|
|
200
|
+
catch (err) { throwUsage(`version file not valid JSON: ${err.message}`); }
|
|
201
|
+
const registryPath = flags.registry ? resolve(flags.registry) : registryPathFromEnv();
|
|
202
|
+
const { registry: loaded, errors } = loadRegistry(registryPath);
|
|
203
|
+
if (errors.length) {
|
|
204
|
+
console.error("Registry errors (refusing to register):");
|
|
205
|
+
for (const e of errors) console.error(` - ${e}`);
|
|
206
|
+
process.exit(2);
|
|
207
|
+
}
|
|
208
|
+
const registry = loaded.specialists ? loaded : emptyRegistry();
|
|
209
|
+
let entry = registry.specialists.find((s) => s.role === role);
|
|
210
|
+
if (!entry) {
|
|
211
|
+
if (typeof flags["backend-url"] !== "string") {
|
|
212
|
+
throwUsage(`new role "${role}": pass --backend-url <url>`);
|
|
213
|
+
}
|
|
214
|
+
entry = {
|
|
215
|
+
role,
|
|
216
|
+
backend_url: flags["backend-url"],
|
|
217
|
+
fallback: "claude",
|
|
218
|
+
workload_quota: Number.isFinite(Number(flags["workload-quota"])) ? Number(flags["workload-quota"]) : 0.7,
|
|
219
|
+
active_version: null,
|
|
220
|
+
versions: [],
|
|
221
|
+
};
|
|
222
|
+
registry.specialists.push(entry);
|
|
223
|
+
}
|
|
224
|
+
if (entry.versions.find((v) => v.id === version.id)) {
|
|
225
|
+
throwUsage(`version id "${version.id}" already exists for role "${role}" (R3)`);
|
|
226
|
+
}
|
|
227
|
+
entry.versions.push(version);
|
|
228
|
+
|
|
229
|
+
saveRegistry(registryPath, registry); // re-validates the whole file; R1/R3/R4/R5/R6/R7 enforced here
|
|
230
|
+
|
|
231
|
+
const eventsPath = flags.events ? resolve(flags.events) : eventsPathFromEnv();
|
|
232
|
+
appendEvent(eventsPath, {
|
|
233
|
+
kind: "register",
|
|
234
|
+
role,
|
|
235
|
+
ts: new Date().toISOString(),
|
|
236
|
+
data: { version_id: version.id, certified_level: version.certified_level, operator: flags.operator || "(unknown)" },
|
|
237
|
+
});
|
|
238
|
+
console.log(`registered ${role}/${version.id} (${version.certified_level}) — active_version unchanged (${entry.active_version || "null"}).`);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// ── promote ─────────────────────────────────────────────────────────────────────────────────
|
|
242
|
+
|
|
243
|
+
function promoteSpecialist(args) {
|
|
244
|
+
const { flags, positional } = parseArgs(args);
|
|
245
|
+
const role = positional[0];
|
|
246
|
+
const versionId = positional[1];
|
|
247
|
+
if (!role || !versionId) throwUsage("promote <role> <version-id>");
|
|
248
|
+
return pointerSwap({ role, versionId, flags, kind: "promote" });
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ── rollback ────────────────────────────────────────────────────────────────────────────────
|
|
252
|
+
|
|
253
|
+
function rollbackSpecialist(args) {
|
|
254
|
+
const { flags, positional } = parseArgs(args);
|
|
255
|
+
const role = positional[0];
|
|
256
|
+
const versionId = positional[1];
|
|
257
|
+
if (!role || !versionId) throwUsage("rollback <role> <version-id>");
|
|
258
|
+
return pointerSwap({ role, versionId, flags, kind: "rollback" });
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Shared pointer-swap routine for promote and rollback. The mechanism is identical (set
|
|
263
|
+
* active_version); the SEMANTICS differ:
|
|
264
|
+
* - promote: a NEW version becomes active (forward); typically gated by certification.
|
|
265
|
+
* - rollback: a PRIOR version becomes active (backward); the NAMED COMPENSATOR.
|
|
266
|
+
* The event log records the kind so the audit trail distinguishes them.
|
|
267
|
+
*/
|
|
268
|
+
function pointerSwap({ role, versionId, flags, kind }) {
|
|
269
|
+
const registryPath = flags.registry ? resolve(flags.registry) : registryPathFromEnv();
|
|
270
|
+
const { registry, errors } = loadRegistry(registryPath);
|
|
271
|
+
if (errors.length) {
|
|
272
|
+
console.error("Registry errors (refusing to mutate):");
|
|
273
|
+
for (const e of errors) console.error(` - ${e}`);
|
|
274
|
+
process.exit(2);
|
|
275
|
+
}
|
|
276
|
+
const entry = registry.specialists.find((s) => s.role === role);
|
|
277
|
+
if (!entry) {
|
|
278
|
+
throwUsage(`role "${role}" not found in registry`);
|
|
279
|
+
}
|
|
280
|
+
const version = entry.versions.find((v) => v.id === versionId);
|
|
281
|
+
if (!version) {
|
|
282
|
+
throwUsage(`version "${versionId}" not found for role "${role}" (R4)`);
|
|
283
|
+
}
|
|
284
|
+
if (version.certified_level === "L0") {
|
|
285
|
+
const err = new Error(`refusing to ${kind} an L0 (uncertified) version: ${role}/${versionId}`);
|
|
286
|
+
err.exitCode = 2;
|
|
287
|
+
err.hint = "Reject 2: an uncertified specialist cannot be active. Train and certify first.";
|
|
288
|
+
throw err;
|
|
289
|
+
}
|
|
290
|
+
const previous = entry.active_version;
|
|
291
|
+
entry.active_version = versionId;
|
|
292
|
+
saveRegistry(registryPath, registry);
|
|
293
|
+
|
|
294
|
+
const eventsPath = flags.events ? resolve(flags.events) : eventsPathFromEnv();
|
|
295
|
+
appendEvent(eventsPath, {
|
|
296
|
+
kind,
|
|
297
|
+
role,
|
|
298
|
+
ts: new Date().toISOString(),
|
|
299
|
+
data: {
|
|
300
|
+
from_version: previous,
|
|
301
|
+
to_version: versionId,
|
|
302
|
+
certified_level: version.certified_level,
|
|
303
|
+
operator: flags.operator || "(unknown)",
|
|
304
|
+
reason: flags.reason || "",
|
|
305
|
+
},
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
const arrow = kind === "rollback" ? "←" : "→";
|
|
309
|
+
console.log(`${kind} ${role}: ${previous || "(none)"} ${arrow} ${versionId} (${version.certified_level})`);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// ── clear-halt ──────────────────────────────────────────────────────────────────────────────
|
|
313
|
+
|
|
314
|
+
function clearHaltSpecialist(args) {
|
|
315
|
+
const { flags, positional } = parseArgs(args);
|
|
316
|
+
const role = positional[0];
|
|
317
|
+
if (!role) throwUsage("clear-halt <role>");
|
|
318
|
+
const statePath = flags.state ? resolve(flags.state) : statePathFromEnv();
|
|
319
|
+
const eventsPath = flags.events ? resolve(flags.events) : eventsPathFromEnv();
|
|
320
|
+
let state;
|
|
321
|
+
try { state = loadState(statePath); }
|
|
322
|
+
catch { state = emptyState(); }
|
|
323
|
+
const halt = getHalt(state, role);
|
|
324
|
+
if (!halt.halted) {
|
|
325
|
+
console.log(`role "${role}" was not halted; no action taken.`);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
setHalt(state, role, null);
|
|
329
|
+
saveState(statePath, state);
|
|
330
|
+
const ts = new Date().toISOString();
|
|
331
|
+
appendClearHaltEvent(eventsPath, {
|
|
332
|
+
role,
|
|
333
|
+
ts,
|
|
334
|
+
operator: flags.operator || "(unknown)",
|
|
335
|
+
reason: flags.reason || "",
|
|
336
|
+
});
|
|
337
|
+
console.log(`cleared halt for "${role}". previous reason: ${halt.reason}`);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// ── helpers ─────────────────────────────────────────────────────────────────────────────────
|
|
341
|
+
|
|
342
|
+
function registryPathFromEnv() {
|
|
343
|
+
return resolve(process.env.ROLEOS_SPECIALISTS_PATH || DEFAULT_REGISTRY_PATH);
|
|
344
|
+
}
|
|
345
|
+
function statePathFromEnv() {
|
|
346
|
+
return resolve(process.env.ROLEOS_SPECIALIST_STATE_PATH || DEFAULT_STATE_PATH);
|
|
347
|
+
}
|
|
348
|
+
function eventsPathFromEnv() {
|
|
349
|
+
return resolve(process.env.ROLEOS_SPECIALIST_EVENTS_PATH || DEFAULT_EVENTS_PATH);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function parseArgs(args) {
|
|
353
|
+
const flags = {};
|
|
354
|
+
const positional = [];
|
|
355
|
+
for (let i = 0; i < args.length; i++) {
|
|
356
|
+
const a = args[i];
|
|
357
|
+
if (a.startsWith("--")) {
|
|
358
|
+
const name = a.slice(2);
|
|
359
|
+
const next = args[i + 1];
|
|
360
|
+
if (next !== undefined && !next.startsWith("--")) {
|
|
361
|
+
flags[name] = next;
|
|
362
|
+
i += 1;
|
|
363
|
+
} else {
|
|
364
|
+
flags[name] = true;
|
|
365
|
+
}
|
|
366
|
+
} else {
|
|
367
|
+
positional.push(a);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return { flags, positional };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function throwUsage(detail) {
|
|
374
|
+
const err = new Error(`usage: ${detail}`);
|
|
375
|
+
err.exitCode = 1;
|
|
376
|
+
err.hint = "Run 'roleos specialist help' for the full subcommand list.";
|
|
377
|
+
throw err;
|
|
378
|
+
}
|
package/src/verify-citations.mjs
CHANGED
|
@@ -196,6 +196,7 @@ export function gateCitations(prismResponse) {
|
|
|
196
196
|
detail: cr.detail,
|
|
197
197
|
span: cr.supporting_span ?? null,
|
|
198
198
|
source_title: cr.source_title ?? null,
|
|
199
|
+
source_abstract: cr.source_abstract ?? null,
|
|
199
200
|
}));
|
|
200
201
|
// role-os enforces the deterministic floor ITSELF (it does not delegate it to prism's top-level
|
|
201
202
|
// aggregation): any fabricated-existence citation BLOCKS and dominates a top-level "accept", so a
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
# Specialist Tier — policy
|
|
2
|
+
|
|
3
|
+
This file is law, not description. It governs when Role OS routes a dispatch to a **specialist**
|
|
4
|
+
(a locally-served, trained low-rank adapter on a non-Claude base model) instead of to Claude. The
|
|
5
|
+
tier ships with reject conditions — a specialist that cannot be told "no" is not done
|
|
6
|
+
(role-os-lockdown-doctrine, §5).
|
|
7
|
+
|
|
8
|
+
v0.1 ships the FRAMEWORK only. No specialist is trained or deployed; the control plane is proven
|
|
9
|
+
end-to-end with a stub backend. Real specialists arrive through their own training kickoffs.
|
|
10
|
+
|
|
11
|
+
## Glossary (office language — no RPG terms)
|
|
12
|
+
|
|
13
|
+
| Term | Meaning |
|
|
14
|
+
|------|---------|
|
|
15
|
+
| role | a job a specialist can be trained for (existing Role OS concept) |
|
|
16
|
+
| specialist | a trained, versioned adapter deployed for a role |
|
|
17
|
+
| certification level | a training round that passed the eval gate (e.g. "Verifier, certified L2") |
|
|
18
|
+
| certification exam | the frozen labeled eval set |
|
|
19
|
+
| field audit | the rolling production-slice eval |
|
|
20
|
+
| dispatch criteria | the rule for when to use a specialist |
|
|
21
|
+
| cross-training | a specialist trained for two roles, dispatched in sequence (never fused weights) |
|
|
22
|
+
| version rollback | revert to a previous certified specialist version |
|
|
23
|
+
| workload quota | the maximum share of dispatches one specialist may take in a window |
|
|
24
|
+
| gate | the separate dispatcher that decides specialist-vs-Claude per dispatch |
|
|
25
|
+
|
|
26
|
+
The earlier RPG framing (class / character / XP / level / multiclass / party / respec) is dropped.
|
|
27
|
+
Use these office terms in code, policy, docs, and comments.
|
|
28
|
+
|
|
29
|
+
## Standards compliance (required — workflow-standards.md)
|
|
30
|
+
|
|
31
|
+
| # | Standard | Score | Evidence |
|
|
32
|
+
|---|----------|-------|----------|
|
|
33
|
+
| 1 | PIN_PER_STEP | 2 | A specialist dispatch pins `adapter_id`, `base_model`, `gate_threshold`, and `exam_hash` in the registry entry and the dispatch log; replayable. The v0.1 gate is deterministic + embedding-similarity (no model pin needed yet). Remediation to 3: when the gate becomes a trained classifier, pin `gate_model_id` + `gate_version` as well. Owner: tier maintainer. Target: later kickoff that introduces the trained gate. |
|
|
34
|
+
| 2 | ANDON_AUTHORITY | 3 | The gate fails open to Claude on any uncertainty (low OvA score ∨ OOD ∨ quota-exhausted ∨ backend unreachable). Shadow-probe disagreement above τ halts specialist dispatch for the affected role and emits an andon event consumed by `roleos specialist status`. The reject conditions are enforced in `src/specialist/gate.mjs`, not described in prose. |
|
|
35
|
+
| 3 | NAMED_COMPENSATORS | 3 | `roleos specialist rollback <role> <version>` is the named compensator — a pure pointer swap in the registry to a prior certified adapter, no retrain. Owner: tier maintainer. See the Compensators table below. v0.1 performs no irreversible external action — no publish, no release, no tag. |
|
|
36
|
+
| 4 | DECOMPOSE_BY_SECRETS | 3 | Four boundaries, each hiding one secret family (Parnas CACM 1972): `gate` hides routing math; `client` hides the vLLM HTTP wire format; `registry` hides the on-disk schema; `policy` hides the dispatch law. Changing one does not ripple. |
|
|
37
|
+
| 5 | UNCERTAINTY_GATED_HUMANS | 2 | Shadow-probe disagreement above τ writes a director checkpoint with a contrastive message ("the specialist said X; Claude said Y; I halted this role because the rolling window disagreement is Z > τ"). Held at 2 — v0.1 logs the checkpoint, it is not yet interactive. Remediation to 3: wire the checkpoint to the director-review channel. Owner: tier maintainer. Target: v0.2. |
|
|
38
|
+
| 6 | EXTERNAL_VERIFIER | 3 | The specialist base model is cross-family (Qwen3 / Gemma) by construction; satisfies #6 natively (Panickssery NeurIPS 2024, arXiv:2404.13076). When a specialist's verdict is consumed by a verifier (e.g. the Verifier specialist plugs into prism's L4 lens), prism's own family-different + submodularity + strip guards still wrap the call — the specialist does not get to bypass them. |
|
|
39
|
+
|
|
40
|
+
## Compensators (no skip)
|
|
41
|
+
|
|
42
|
+
v0.1 performs **no** irreversible external action: no `npm publish`, no `gh release create`, no
|
|
43
|
+
`git push <tag>`, no `gh repo edit`, no GitHub Pages deploy, no posted PR, no sent email, no
|
|
44
|
+
charged card, no external DB write. The only irreversible-ish action in this kickoff is a `git
|
|
45
|
+
commit`, undone by `git revert` or branch deletion.
|
|
46
|
+
|
|
47
|
+
The tier's in-product irreversible-ish actions and their compensators:
|
|
48
|
+
|
|
49
|
+
| Action | What it does | Compensator | Owner | Post-rollback state |
|
|
50
|
+
|--------|--------------|-------------|-------|---------------------|
|
|
51
|
+
| Promote a specialist to active for a role | Sets the registry's `active_version` pointer for the role | `roleos specialist rollback <role> <version>` — pointer swap to a prior certified version; no retrain, no adapter delete | tier maintainer | The role routes to the rolled-back specialist version on the next dispatch. The previously-active version remains in the registry's history and is itself rollback-targetable. |
|
|
52
|
+
| Workload-quota halt | Stops further specialist dispatches for a role this window | (no compensator needed — halt is a local in-memory cap that resets at the next window boundary) | tier maintainer | Next window opens with a fresh quota; no state needs unwinding. |
|
|
53
|
+
| Shadow-probe-disagreement halt (andon) | Stops further specialist dispatches for a role until cleared | `roleos specialist clear-halt <role> [--reason …]` — clears the halt, logs the operator + reason to the receipt | tier maintainer | The role resumes specialist dispatch on the next call. The halt event remains in the shadow-probe log. |
|
|
54
|
+
|
|
55
|
+
A specialist that cannot be rolled back is not certified. A role whose halt cannot be cleared is
|
|
56
|
+
not operable. Both compensators are pure pointer-state changes — no irreversible side effects.
|
|
57
|
+
|
|
58
|
+
## What a specialist is
|
|
59
|
+
|
|
60
|
+
A specialist is:
|
|
61
|
+
|
|
62
|
+
- A **low-rank adapter** (rank 8–16, α = 2r, q/v/o projections — see specialist-tier-architecture
|
|
63
|
+
Lock 1) trained for one role.
|
|
64
|
+
- **Versioned** in the role-os specialist registry (`.role-os/specialists.json` by default).
|
|
65
|
+
- **Served** over HTTP by a vLLM multi-LoRA backend (or any HTTP service implementing the
|
|
66
|
+
contract in [Specialist HTTP contract](#specialist-http-contract)). The protocol is
|
|
67
|
+
language-agnostic; v0.1's stub backend is a local Node handler returning canned verdicts.
|
|
68
|
+
- **Base model cross-family from Claude** — Qwen3 / Gemma in v1 (Lock 5). A Claude-family base
|
|
69
|
+
model is rejected at registry load time.
|
|
70
|
+
- **Subordinate to the consumer's own checks.** When the Verifier specialist plugs into
|
|
71
|
+
prism's L4 lens, prism's family-different routing, reasoning strip, and submodularity guards
|
|
72
|
+
still apply. The specialist tier does not grant any consumer permission to skip those.
|
|
73
|
+
|
|
74
|
+
A specialist is **not**:
|
|
75
|
+
|
|
76
|
+
- A general-purpose model. Specialists are narrow by design (Snell 2024, arXiv:2411.16035 —
|
|
77
|
+
narrow fine-tunes phase-transition; capacity beyond the role is wasted).
|
|
78
|
+
- A judge of its own confidence. The dispatch decision belongs to a **separate gate** (Verma &
|
|
79
|
+
Nalisnick ICML 2022, arXiv:2202.03673; Leng 2024, arXiv:2410.09724 — RLHF inflates verbalized
|
|
80
|
+
confidence).
|
|
81
|
+
- A fused-multiclass blob. Two roles trained together stay as two specialists dispatched in
|
|
82
|
+
sequence (Chen et al. 2025, arXiv:2506.13479; Zhang & Zhou 2025, arXiv:2505.22934). Multiclass
|
|
83
|
+
is a v2 research bet behind an A/B gate.
|
|
84
|
+
|
|
85
|
+
## Dispatch criteria
|
|
86
|
+
|
|
87
|
+
For every dispatch where a role has an active certified specialist:
|
|
88
|
+
|
|
89
|
+
1. **OvA score**. The gate computes `P(specialist is right)` as a One-vs-All score, per
|
|
90
|
+
specialist (Verma & Nalisnick ICML 2022). A joint softmax across roles is rejected — it
|
|
91
|
+
couples specialists that are independently trained.
|
|
92
|
+
2. **OOD check**. The gate computes a deterministic out-of-distribution signal against the
|
|
93
|
+
specialist's `exam_hash`-pinned input distribution. Any input outside the band is OOD.
|
|
94
|
+
3. **Workload quota**. The gate tracks the share of dispatches the specialist has taken in the
|
|
95
|
+
current window. A specialist at or above its quota is treated as quota-exhausted.
|
|
96
|
+
4. **Route decision**:
|
|
97
|
+
- `score > gate_threshold ∧ ¬OOD ∧ ¬quota-exhausted` → route to the specialist.
|
|
98
|
+
- Any other case → route to Claude (the role's existing fallback).
|
|
99
|
+
5. **Shadow probe**. Every Kth specialist dispatch additionally runs the Claude path. The two
|
|
100
|
+
results are compared and logged. Rolling-window disagreement above τ halts specialist
|
|
101
|
+
dispatch for the role and emits an andon event.
|
|
102
|
+
|
|
103
|
+
The gate **fails open to Claude**. Any uncertainty, any backend failure, any malformed response,
|
|
104
|
+
any OOD input is a fail-open. A mis-routed specialist must never silently corrupt downstream
|
|
105
|
+
work.
|
|
106
|
+
|
|
107
|
+
## Reject conditions (the system must say no)
|
|
108
|
+
|
|
109
|
+
These are hard gates enforced in code, not guidelines. A new specialist that violates any of
|
|
110
|
+
them is rejected at registry load or at dispatch time.
|
|
111
|
+
|
|
112
|
+
1. **Same-family base.** If `base_model` resolves to a Claude-family identifier the registry
|
|
113
|
+
refuses to load the entry. Workflow-standard #6 is satisfied by construction; same-family is
|
|
114
|
+
a correctness regression, not a routing preference.
|
|
115
|
+
2. **Uncertified specialist promoted to active.** A specialist with no `certified_level` cannot
|
|
116
|
+
be set as `active_version`. The registry refuses the promotion.
|
|
117
|
+
3. **Workload over quota.** If the rolling-window dispatch share for a specialist exceeds its
|
|
118
|
+
declared `workload_quota`, the gate refuses to route to it until the window rolls. No
|
|
119
|
+
bypass flag.
|
|
120
|
+
4. **OvA score under threshold.** If `score < gate_threshold`, the gate refuses to route to
|
|
121
|
+
the specialist. No "low confidence override."
|
|
122
|
+
5. **OOD input.** If the gate's OOD check fires, the gate refuses to route. No "trust the
|
|
123
|
+
specialist on OOD" flag.
|
|
124
|
+
6. **Backend unreachable.** If the specialist backend returns no parseable response within the
|
|
125
|
+
per-call timeout, the gate refuses to use the result (fail-open to Claude). An unreachable
|
|
126
|
+
gate is a closed gate — same invariant as the citation gate.
|
|
127
|
+
7. **Shadow-probe disagreement above τ.** If the rolling-window disagreement between the
|
|
128
|
+
specialist and Claude exceeds τ on the same input distribution, dispatch to the specialist
|
|
129
|
+
is halted for the role until operator clears via `roleos specialist clear-halt`.
|
|
130
|
+
8. **Consumer's own guards rejected the result.** If the consumer (e.g. prism on the Verifier
|
|
131
|
+
specialist's L4 verdict) rejects the result via its own checks, the result is not accepted
|
|
132
|
+
regardless of OvA score. The specialist tier does not override consumer-side guards.
|
|
133
|
+
|
|
134
|
+
## Workload quota
|
|
135
|
+
|
|
136
|
+
Each registry entry declares a `workload_quota` (a decimal in `(0, 1]`) — the maximum share of
|
|
137
|
+
dispatches the specialist may take in a sliding window. v0.1 defaults: window = 200 dispatches,
|
|
138
|
+
quota = 0.7. The quota is a hard cap; the gate refuses to route to a quota-exhausted specialist
|
|
139
|
+
until the window rolls (Switch Transformer load-balance pressure — Fedus et al. 2022,
|
|
140
|
+
arXiv:2101.03961).
|
|
141
|
+
|
|
142
|
+
The quota exists for two reasons:
|
|
143
|
+
|
|
144
|
+
- **Anti-collapse.** Without a cap, a slightly-better specialist takes 100% of dispatches and
|
|
145
|
+
the production distribution drifts off the specialist's training distribution. The quota
|
|
146
|
+
forces enough Claude traffic to keep the shadow-probe distribution honest.
|
|
147
|
+
- **Operator review.** A specialist at the quota limit is a signal that the role's load
|
|
148
|
+
pattern matched the training distribution well — and that any quality slip will not be hidden
|
|
149
|
+
in the noise of a small specialist sample.
|
|
150
|
+
|
|
151
|
+
The quota window resets on a sliding basis (not aligned to wall clock) so a burst of dispatches
|
|
152
|
+
cannot be timed against a window edge.
|
|
153
|
+
|
|
154
|
+
## Shadow probe
|
|
155
|
+
|
|
156
|
+
Every Kth specialist dispatch (default K = 20) additionally runs the Claude path. The two
|
|
157
|
+
results are logged with the dispatch id and an agreement bit. Over a rolling window of N probes
|
|
158
|
+
(default N = 50), if the agreement rate falls below `1 - τ` (default τ = 0.15), the role is
|
|
159
|
+
halted:
|
|
160
|
+
|
|
161
|
+
- Halt writes a contrastive director checkpoint: "the specialist said X; Claude said Y;
|
|
162
|
+
rolling-window disagreement is Z > τ; specialist dispatch for role R halted; clear with
|
|
163
|
+
`roleos specialist clear-halt R`."
|
|
164
|
+
- The halt is sticky — it survives across processes and is cleared only by explicit operator
|
|
165
|
+
action.
|
|
166
|
+
|
|
167
|
+
Shadow probes do not affect the dispatch's user-visible result — Claude is called *in addition
|
|
168
|
+
to* the specialist, not instead. The user sees the specialist's verdict (or the gate's
|
|
169
|
+
fail-open verdict from Claude); the probe is a background quality check.
|
|
170
|
+
|
|
171
|
+
## Certification
|
|
172
|
+
|
|
173
|
+
A specialist is **certified at level N** only when:
|
|
174
|
+
|
|
175
|
+
1. **Certification exam** (the frozen labeled eval set) shows non-overlapping confidence
|
|
176
|
+
intervals between this version and the previous certified version, in the direction of
|
|
177
|
+
improvement.
|
|
178
|
+
2. **Field audit** (the rolling production-slice eval) agrees with the exam in the same
|
|
179
|
+
direction over a configured window. Exam-vs-audit divergence is the overfitting /
|
|
180
|
+
contamination alarm (DICE, Ye 2024).
|
|
181
|
+
3. **Two-seed replication.** The same training recipe with a different seed produces a
|
|
182
|
+
specialist that hits the same level. Narrow fine-tunes show phase-transition behavior — a
|
|
183
|
+
single-seed result is not a level.
|
|
184
|
+
|
|
185
|
+
Level is **monotone**: a specialist certified at L2 stays at L2 even if a later version fails.
|
|
186
|
+
A failed level attempt does not demote a prior version; it just means the new version did not
|
|
187
|
+
earn the new level.
|
|
188
|
+
|
|
189
|
+
The eval harness (certification exam + field audit + replication) is built by the training
|
|
190
|
+
kickoffs, not in v0.1. v0.1 specifies the contract.
|
|
191
|
+
|
|
192
|
+
## Version rollback
|
|
193
|
+
|
|
194
|
+
`roleos specialist rollback <role> <version>` swaps the registry's `active_version` pointer
|
|
195
|
+
for the role to the named prior version. No retrain. No adapter delete. No data migration.
|
|
196
|
+
Rollback is a pure pointer change.
|
|
197
|
+
|
|
198
|
+
The previously-active version stays in the registry's `versions` list — it is itself
|
|
199
|
+
rollback-targetable. A rollback emits a receipt with the operator, the from-version, the
|
|
200
|
+
to-version, and the reason.
|
|
201
|
+
|
|
202
|
+
A version that cannot be rolled back is not certified. A registry that does not preserve prior
|
|
203
|
+
versions is broken.
|
|
204
|
+
|
|
205
|
+
## Specialist HTTP contract
|
|
206
|
+
|
|
207
|
+
A specialist backend implements one endpoint:
|
|
208
|
+
|
|
209
|
+
```
|
|
210
|
+
POST <backend_url>/verify
|
|
211
|
+
Content-Type: application/json
|
|
212
|
+
|
|
213
|
+
Request:
|
|
214
|
+
{
|
|
215
|
+
"adapter_id": "<string — the trained adapter pin>",
|
|
216
|
+
"role": "<string — the role name, for logging>",
|
|
217
|
+
"input": <object — role-specific schema, opaque to the gate>,
|
|
218
|
+
"trace_id": "<string — propagated for receipts>"
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
Response (200):
|
|
222
|
+
{
|
|
223
|
+
"verdict": <object — role-specific verdict, opaque to the gate>,
|
|
224
|
+
"score": <number in [0, 1] — the specialist's self-reported score, INFORMATIONAL>,
|
|
225
|
+
"adapter_id": "<string — echo of the adapter pin>",
|
|
226
|
+
"base_model": "<string — echo of the base model, for receipts>",
|
|
227
|
+
"duration_ms": <number>
|
|
228
|
+
}
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Notes:
|
|
232
|
+
|
|
233
|
+
- The specialist's self-reported `score` is **informational** — it does not enter the routing
|
|
234
|
+
decision (the gate has its own OvA score). It is logged to the receipt for drift analysis.
|
|
235
|
+
- The `adapter_id` echo lets the gate verify the backend served the pinned adapter, not a
|
|
236
|
+
different one with the same role.
|
|
237
|
+
- Non-200, non-JSON, or timeout = the gate fails open to Claude.
|
|
238
|
+
- The contract is language-agnostic. The v0.1 stub backend is a local Node handler returning
|
|
239
|
+
canned verdicts; the v1 backend is vLLM multi-LoRA (S-LoRA, Sheng 2024) via gpu-container.
|
|
240
|
+
|
|
241
|
+
## What v0.1 is and is not
|
|
242
|
+
|
|
243
|
+
**v0.1 ships:**
|
|
244
|
+
|
|
245
|
+
- This policy file (the law).
|
|
246
|
+
- The role schema extension (the `specialist:` block).
|
|
247
|
+
- The registry (load + validate `.role-os/specialists.json`).
|
|
248
|
+
- The gate (OvA + OOD + quota + threshold + fail-open).
|
|
249
|
+
- The HTTP client (POST to `backend_url`, injectable for tests).
|
|
250
|
+
- The shadow-probe scheduler + halt.
|
|
251
|
+
- The `roleos specialist` CLI surface (status / list / rollback / clear-halt).
|
|
252
|
+
- Unit tests for each reject condition + one end-to-end test against an in-process stub backend.
|
|
253
|
+
|
|
254
|
+
**v0.1 does not ship:**
|
|
255
|
+
|
|
256
|
+
- Any trained adapter. The Verifier specialist (`KICKOFF-specialist-verifier-dataset.md`) and
|
|
257
|
+
the Token Budget Analyst (`KICKOFF-specialist-token-budget-dataset.md`) build the corpora;
|
|
258
|
+
training happens in `KICKOFF-specialist-training.md`.
|
|
259
|
+
- A trained gate classifier. The v0.1 gate is deterministic + embedding-similarity; a trained
|
|
260
|
+
gate is an additive upgrade behind PIN_PER_STEP.
|
|
261
|
+
- Real vLLM serving. v0.1 talks to a stub backend over HTTP; the real vLLM backend lives in
|
|
262
|
+
gpu-container.
|
|
263
|
+
- Ollama integration. Ollama cannot per-request hot-swap adapters (issue #9548); ollama-intern
|
|
264
|
+
wiring is a later, separate concern (Lock 2 of specialist-tier-architecture).
|
|
265
|
+
- Fused multiclass. Sequential dispatch only (Lock 4).
|
|
266
|
+
- An interactive director checkpoint. v0.1 logs the contrastive message; v0.2 wires it to the
|
|
267
|
+
review channel.
|
|
268
|
+
|
|
269
|
+
## References
|
|
270
|
+
|
|
271
|
+
- specialist-tier-architecture.md — the locked decisions this policy enforces.
|
|
272
|
+
- role-os-lockdown-doctrine.md — why this tier ships with reject conditions, not scaffolding.
|
|
273
|
+
- workflow-standards.md — the six standards every workflow must score.
|
|
274
|
+
- prism-verify design/01-research-grounding.md — Locks 1–4 the Verifier consumer enforces.
|
|
275
|
+
- Panickssery, Bowman & Feng — *LLM Evaluators Recognize and Favor Their Own Generations*,
|
|
276
|
+
NeurIPS 2024, https://arxiv.org/abs/2404.13076 (cross-family base, #6).
|
|
277
|
+
- Verma & Nalisnick — *Calibration of Selective Classifiers via One-vs-All Scores*, ICML 2022,
|
|
278
|
+
https://arxiv.org/abs/2202.03673 (OvA gate, no joint softmax).
|
|
279
|
+
- Leng et al. — *Taming Overconfidence in LLMs*, 2024, https://arxiv.org/abs/2410.09724
|
|
280
|
+
(separate gate, no self-report).
|
|
281
|
+
- Fedus, Zoph & Shazeer — *Switch Transformers*, 2022, https://arxiv.org/abs/2101.03961
|
|
282
|
+
(load-balance pressure → workload quota).
|
|
283
|
+
- Chen et al. 2025, https://arxiv.org/abs/2506.13479 — weight merging fails for adjacent
|
|
284
|
+
narrow specialists (sequential dispatch, not fused multiclass).
|
|
285
|
+
- Snell et al. 2024, https://arxiv.org/abs/2411.16035 — narrow fine-tunes phase-transition
|
|
286
|
+
(two-seed replication required for a certification level).
|
|
287
|
+
- Buçinca et al. — *Contrastive Explanations*, CHI 2025, https://arxiv.org/abs/2410.04253
|
|
288
|
+
(the shadow-probe halt's contrastive message).
|