ultra-builder-pro-cli 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -7
- package/README.md +14 -16
- package/adapters/_shared/codex-assets.cjs +32 -151
- package/adapters/_shared/frontmatter.cjs +2 -2
- package/adapters/_shared/handbook.cjs +3 -3
- package/adapters/_shared/path-rewrite.cjs +0 -3
- package/adapters/_shared/runtime-assets.cjs +11 -4
- package/adapters/_shared/tests/path-rewrite.test.cjs +1 -3
- package/adapters/_shared/tests/resolve-target.test.cjs +1 -11
- package/adapters/_shared/tests/runtime-assets.test.cjs +17 -6
- package/adapters/opencode.js +1 -6
- package/adapters/tests/claude.test.cjs +1 -4
- package/adapters/tests/codex.test.cjs +5 -6
- package/adapters/tests/opencode.test.cjs +1 -2
- package/agents/tdd-runner.md +4 -3
- package/bin/install.js +6 -9
- package/bin/orchestrator.js +11 -4
- package/docs/AGENT-CONTEXT.md +4 -5
- package/docs/ARCHITECTURE.md +3 -4
- package/docs/LEGACY-HERMES.md +5 -81
- package/docs/PLAN.zh-CN.md +35 -42
- package/docs/ROADMAP.md +3 -3
- package/docs/RUNTIME-COMPAT-MATRIX.md +61 -60
- package/mcp-server/lib/pricing.cjs +0 -6
- package/mcp-server/lib/state-ops.cjs +7 -0
- package/mcp-server/lib/telemetry.cjs +4 -0
- package/mcp-server/tests/server-contract.test.cjs +17 -0
- package/mcp-server/tests/state-ops.test.cjs +23 -0
- package/mcp-server/tests/telemetry.test.cjs +14 -0
- package/orchestrator/dispatch-rules.cjs +15 -4
- package/orchestrator/tests/daemon.test.cjs +13 -12
- package/orchestrator/tests/dispatch-rules.test.cjs +12 -10
- package/package.json +3 -5
- package/skills/ultra-init/SKILL.md +0 -1
- package/skills/ultra-verify/SKILL.md +22 -41
- package/skills/ultra-verify/evals/evals.json +22 -24
- package/skills/ultra-verify/references/confidence-system.md +15 -32
- package/skills/ultra-verify/references/cross-verify-modes.md +14 -59
- package/skills/ultra-verify/scripts/verify_wait.py +44 -81
- package/spec/mcp-tools.yaml +3 -3
- package/spec/schemas/skill-manifest.schema.json +2 -2
- package/spec/schemas/state-db.sql +2 -2
- package/adapters/_shared/md-to-toml.cjs +0 -54
- package/adapters/_shared/tests/md-to-toml.test.cjs +0 -48
- package/adapters/gemini.js +0 -204
- package/adapters/tests/gemini.test.cjs +0 -112
- package/skills/gemini-collab/SKILL.md +0 -51
- package/skills/gemini-collab/evals/evals.json +0 -42
|
@@ -13,9 +13,11 @@
|
|
|
13
13
|
// parallel orchestrator (8B.2) gets a richer decision surface: wave state,
|
|
14
14
|
// deps readiness, and custom rule injection.
|
|
15
15
|
|
|
16
|
+
const { isSupportedRuntime } = require('../adapters/_shared/runtime-assets.cjs');
|
|
17
|
+
|
|
16
18
|
const ROUTE_PREFERENCES = Object.freeze({
|
|
17
|
-
haiku: ['opencode', '
|
|
18
|
-
sonnet: ['claude', 'codex', 'opencode'
|
|
19
|
+
haiku: ['opencode', 'claude', 'codex'],
|
|
20
|
+
sonnet: ['claude', 'codex', 'opencode'],
|
|
19
21
|
opus: ['claude', 'codex'],
|
|
20
22
|
});
|
|
21
23
|
|
|
@@ -73,12 +75,21 @@ const DEFAULT_RULES = Object.freeze([
|
|
|
73
75
|
]);
|
|
74
76
|
|
|
75
77
|
function evaluate(ctx, rules = DEFAULT_RULES) {
|
|
78
|
+
const normalizedCtx = {
|
|
79
|
+
...ctx,
|
|
80
|
+
available_runtimes: Array.isArray(ctx && ctx.available_runtimes)
|
|
81
|
+
? [...new Set(ctx.available_runtimes.filter(isSupportedRuntime))]
|
|
82
|
+
: [],
|
|
83
|
+
};
|
|
76
84
|
const sorted = [...rules].sort((a, b) => (b.priority || 0) - (a.priority || 0));
|
|
77
85
|
for (const rule of sorted) {
|
|
78
|
-
if (rule.when(
|
|
86
|
+
if (rule.when(normalizedCtx)) {
|
|
79
87
|
let runtime = null;
|
|
80
88
|
if (rule.action === 'spawn_agent') {
|
|
81
|
-
runtime = rule.resolve ? rule.resolve(
|
|
89
|
+
runtime = rule.resolve ? rule.resolve(normalizedCtx) : (rule.runtime || null);
|
|
90
|
+
if (!isSupportedRuntime(runtime)) {
|
|
91
|
+
return { rule_id: rule.id, action: 'block', runtime: null };
|
|
92
|
+
}
|
|
82
93
|
}
|
|
83
94
|
return { rule_id: rule.id, action: rule.action, runtime };
|
|
84
95
|
}
|
|
@@ -15,6 +15,7 @@ const { execFileSync } = require('node:child_process');
|
|
|
15
15
|
const { initStateDb, closeStateDb } = require('../../mcp-server/lib/state-db.cjs');
|
|
16
16
|
const ops = require('../../mcp-server/lib/state-ops.cjs');
|
|
17
17
|
const daemon = require('../daemon.cjs');
|
|
18
|
+
const RETIRED_RUNTIME = ['gem', 'ini'].join('');
|
|
18
19
|
|
|
19
20
|
const LONG_SLEEP_CMD = process.execPath;
|
|
20
21
|
const LONG_SLEEP_ARGS = ['-e', 'setInterval(() => {}, 60000);'];
|
|
@@ -55,15 +56,15 @@ function cleanup(repoRoot, db, handle) {
|
|
|
55
56
|
test('routeTask: haiku hint prefers cheap runtimes', () => {
|
|
56
57
|
const runtime = daemon.routeTask(
|
|
57
58
|
{ id: 't', complexity_hint: 'haiku' },
|
|
58
|
-
['claude', 'opencode', 'codex'
|
|
59
|
+
['claude', 'opencode', 'codex'],
|
|
59
60
|
);
|
|
60
|
-
assert.
|
|
61
|
+
assert.equal(runtime, 'opencode');
|
|
61
62
|
});
|
|
62
63
|
|
|
63
64
|
test('routeTask: opus hint prefers strong runtimes', () => {
|
|
64
65
|
const runtime = daemon.routeTask(
|
|
65
66
|
{ id: 't', complexity_hint: 'opus' },
|
|
66
|
-
['claude', 'opencode', 'codex'
|
|
67
|
+
['claude', 'opencode', 'codex'],
|
|
67
68
|
);
|
|
68
69
|
assert.ok(['claude', 'codex'].includes(runtime), `got ${runtime}`);
|
|
69
70
|
});
|
|
@@ -71,23 +72,23 @@ test('routeTask: opus hint prefers strong runtimes', () => {
|
|
|
71
72
|
test('routeTask: sonnet hint middle-ground', () => {
|
|
72
73
|
const runtime = daemon.routeTask(
|
|
73
74
|
{ id: 't', complexity_hint: 'sonnet' },
|
|
74
|
-
['claude', 'opencode', 'codex'
|
|
75
|
+
['claude', 'opencode', 'codex'],
|
|
75
76
|
);
|
|
76
|
-
assert.ok(['claude', 'codex', 'opencode'
|
|
77
|
+
assert.ok(['claude', 'codex', 'opencode'].includes(runtime));
|
|
77
78
|
});
|
|
78
79
|
|
|
79
80
|
test('routeTask: respects availability constraint', () => {
|
|
80
81
|
const runtime = daemon.routeTask(
|
|
81
82
|
{ id: 't', complexity_hint: 'opus' },
|
|
82
|
-
['opencode'
|
|
83
|
+
['opencode'],
|
|
83
84
|
);
|
|
84
85
|
// Prefers claude/codex but those aren't available → fall back to any.
|
|
85
|
-
assert.
|
|
86
|
+
assert.equal(runtime, 'opencode');
|
|
86
87
|
});
|
|
87
88
|
|
|
88
|
-
test('routeTask: no hint
|
|
89
|
-
const runtime = daemon.routeTask({ id: 't' }, [
|
|
90
|
-
assert.
|
|
89
|
+
test('routeTask: no hint skips unsupported runtimes', () => {
|
|
90
|
+
const runtime = daemon.routeTask({ id: 't' }, [RETIRED_RUNTIME, 'claude']);
|
|
91
|
+
assert.equal(runtime, 'claude');
|
|
91
92
|
});
|
|
92
93
|
|
|
93
94
|
test('routeTask: no runtimes available → null', () => {
|
|
@@ -244,7 +245,7 @@ test('runDaemon applies complexity_hint route', async () => {
|
|
|
244
245
|
});
|
|
245
246
|
handle = daemon.runDaemon({
|
|
246
247
|
db, repoRoot,
|
|
247
|
-
runtimes: ['claude', 'opencode'
|
|
248
|
+
runtimes: ['claude', 'opencode'],
|
|
248
249
|
pollMs: 50,
|
|
249
250
|
command: LONG_SLEEP_CMD,
|
|
250
251
|
commandArgs: LONG_SLEEP_ARGS,
|
|
@@ -252,7 +253,7 @@ test('runDaemon applies complexity_hint route', async () => {
|
|
|
252
253
|
await new Promise((r) => setTimeout(r, 300));
|
|
253
254
|
const sessions = db.prepare("SELECT * FROM sessions WHERE task_id = 'd-opus'").all();
|
|
254
255
|
assert.equal(sessions.length, 1);
|
|
255
|
-
// opus should prefer claude over opencode
|
|
256
|
+
// opus should prefer claude over opencode.
|
|
256
257
|
assert.equal(sessions[0].runtime, 'claude');
|
|
257
258
|
} finally {
|
|
258
259
|
cleanup(repoRoot, db, handle);
|
|
@@ -11,12 +11,13 @@ const { test } = require('node:test');
|
|
|
11
11
|
const assert = require('node:assert/strict');
|
|
12
12
|
|
|
13
13
|
const { evaluate, DEFAULT_RULES, ROUTE_PREFERENCES } = require('../dispatch-rules.cjs');
|
|
14
|
+
const RETIRED_RUNTIME = ['gem', 'ini'].join('');
|
|
14
15
|
|
|
15
16
|
function baseCtx(overrides = {}) {
|
|
16
17
|
return {
|
|
17
18
|
task: { id: 't1' },
|
|
18
19
|
deps_ready: true,
|
|
19
|
-
available_runtimes: ['claude', 'opencode', 'codex'
|
|
20
|
+
available_runtimes: ['claude', 'opencode', 'codex'],
|
|
20
21
|
breaker_state: 'ok',
|
|
21
22
|
wave: null,
|
|
22
23
|
...overrides,
|
|
@@ -42,7 +43,7 @@ test('evaluate: opus hint + only codex available → codex', () => {
|
|
|
42
43
|
test('evaluate: opus hint + preference exhausted → first available fallback', () => {
|
|
43
44
|
const d = evaluate(baseCtx({
|
|
44
45
|
task: { id: 't', complexity_hint: 'opus' },
|
|
45
|
-
available_runtimes: ['opencode'
|
|
46
|
+
available_runtimes: ['opencode'],
|
|
46
47
|
}));
|
|
47
48
|
assert.equal(d.action, 'spawn_agent');
|
|
48
49
|
assert.equal(d.runtime, 'opencode');
|
|
@@ -58,13 +59,13 @@ test('evaluate: sonnet hint + all available → claude', () => {
|
|
|
58
59
|
assert.equal(d.runtime, 'claude');
|
|
59
60
|
});
|
|
60
61
|
|
|
61
|
-
test('evaluate:
|
|
62
|
+
test('evaluate: retired runtimes are discarded before fallback routing', () => {
|
|
62
63
|
const d = evaluate(baseCtx({
|
|
63
64
|
task: { id: 't' },
|
|
64
|
-
available_runtimes: [
|
|
65
|
+
available_runtimes: [RETIRED_RUNTIME, 'claude'],
|
|
65
66
|
}));
|
|
66
67
|
assert.equal(d.action, 'spawn_agent');
|
|
67
|
-
assert.equal(d.runtime, '
|
|
68
|
+
assert.equal(d.runtime, 'claude');
|
|
68
69
|
});
|
|
69
70
|
|
|
70
71
|
// ─── block / defer paths (new in 8B.1) ────────────────────────────────────
|
|
@@ -97,20 +98,21 @@ test('evaluate: serial wave already has running task → defer', () => {
|
|
|
97
98
|
|
|
98
99
|
// ─── composability: custom rules override defaults ────────────────────────
|
|
99
100
|
|
|
100
|
-
test('evaluate: custom
|
|
101
|
+
test('evaluate: custom rules cannot select an unsupported runtime', () => {
|
|
101
102
|
const customRules = [
|
|
102
103
|
...DEFAULT_RULES,
|
|
103
104
|
{
|
|
104
|
-
id: 'force-
|
|
105
|
+
id: 'force-retired-runtime',
|
|
105
106
|
priority: 999,
|
|
106
107
|
when: () => true,
|
|
107
108
|
action: 'spawn_agent',
|
|
108
|
-
resolve: () =>
|
|
109
|
+
resolve: () => RETIRED_RUNTIME,
|
|
109
110
|
},
|
|
110
111
|
];
|
|
111
112
|
const d = evaluate(baseCtx({ task: { id: 't', complexity_hint: 'opus' } }), customRules);
|
|
112
|
-
assert.equal(d.
|
|
113
|
-
assert.equal(d.
|
|
113
|
+
assert.equal(d.action, 'block');
|
|
114
|
+
assert.equal(d.runtime, null);
|
|
115
|
+
assert.equal(d.rule_id, 'force-retired-runtime');
|
|
114
116
|
});
|
|
115
117
|
|
|
116
118
|
// ─── ROUTE_PREFERENCES shape guard (regression: don't drift from Phase 5.4) ─
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ultra-builder-pro-cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Multi-runtime installer for the Ultra Builder Pro agent engineering system — deploys commands, agents, skills, hooks, and MCP server to Claude Code, OpenCode,
|
|
3
|
+
"version": "0.6.0",
|
|
4
|
+
"description": "Multi-runtime installer for the Ultra Builder Pro agent engineering system — deploys commands, agents, skills, hooks, and MCP server to Claude Code, OpenCode, and Codex CLI.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"ultra-builder-pro-cli": "bin/install.js",
|
|
7
7
|
"ubp": "bin/install.js",
|
|
@@ -33,7 +33,6 @@
|
|
|
33
33
|
"skills/testing-rules",
|
|
34
34
|
"skills/cc-collab",
|
|
35
35
|
"skills/codex-collab",
|
|
36
|
-
"skills/gemini-collab",
|
|
37
36
|
"skills/ultra-verify",
|
|
38
37
|
"hooks/*.py",
|
|
39
38
|
"hooks/adapters/*.py",
|
|
@@ -60,7 +59,6 @@
|
|
|
60
59
|
"claude-code",
|
|
61
60
|
"opencode",
|
|
62
61
|
"codex",
|
|
63
|
-
"gemini",
|
|
64
62
|
"agent",
|
|
65
63
|
"cli",
|
|
66
64
|
"multi-runtime",
|
|
@@ -87,7 +85,7 @@
|
|
|
87
85
|
"test:state": "node --test 'mcp-server/**/*.test.cjs'",
|
|
88
86
|
"test:orch": "node --test 'orchestrator/**/*.test.cjs'",
|
|
89
87
|
"test:spec": "node spec/scripts/test-all.cjs",
|
|
90
|
-
"test:rest": "node --test adapters/_shared/tests/*.test.cjs adapters/tests/*.test.cjs ultra-tools/*.test.cjs tests/conformance/**/*.test.cjs tests/install.test.cjs tests/package-smoke.test.cjs",
|
|
88
|
+
"test:rest": "node --test adapters/_shared/tests/*.test.cjs adapters/tests/*.test.cjs ultra-tools/*.test.cjs tests/conformance/**/*.test.cjs tests/install.test.cjs tests/package-smoke.test.cjs tests/retired-runtime.test.cjs",
|
|
91
89
|
"test:all": "npm run test:state && npm run test:orch && npm run test:spec && npm run test:rest",
|
|
92
90
|
"test:hooks": "pytest hooks/tests -q",
|
|
93
91
|
"verify:release": "npm run test:all && npm run test:hooks && npm audit --omit=dev --audit-level=high",
|
|
@@ -133,7 +133,6 @@ CLI 最后一行是 `{ "ok": true, "data": { ... } }`(见 spec/cli-protocol.md
|
|
|
133
133
|
| Claude | `/ultra-init [name] [type] [stack] [git]` — 命令薄壳拉起此 skill |
|
|
134
134
|
| OpenCode | `/ultra-init …` 同上(agent frontmatter 小写化) |
|
|
135
135
|
| Codex | `codex exec "run /ultra-init …"`(Phase 4.4 adapter 接) |
|
|
136
|
-
| Gemini | `gemini --prompt "run /ultra-init …"`(Phase 4.5 extension 接) |
|
|
137
136
|
|
|
138
137
|
## 输出锚点
|
|
139
138
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: ultra-verify
|
|
3
|
-
description: Run
|
|
3
|
+
description: Run independent two-model verification with the current host as primary and a read-only Codex advisor, then verify and synthesize the evidence.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
# Ultra Verify —
|
|
6
|
+
# Ultra Verify — Independent Two-Model Verification
|
|
7
7
|
|
|
8
8
|
Claude Code remains primary. It writes the first independent analysis, verifies consequential
|
|
9
|
-
claims, and owns the final answer.
|
|
10
|
-
|
|
9
|
+
claims, and owns the final answer. Codex is an untrusted read-only advisor. Use this workflow only
|
|
10
|
+
when the user explicitly asks for cross-model verification.
|
|
11
11
|
|
|
12
12
|
## Modes
|
|
13
13
|
|
|
@@ -18,11 +18,11 @@ workflow only when the user explicitly asks for cross-model or three-way verific
|
|
|
18
18
|
|
|
19
19
|
## Preconditions
|
|
20
20
|
|
|
21
|
-
1. Confirm `
|
|
21
|
+
1. Confirm `codex --version` plus authentication.
|
|
22
22
|
2. Define the exact workspace, question, evidence standard, and response shape.
|
|
23
23
|
3. Do not send secrets, unrelated files, or an unbounded home directory.
|
|
24
|
-
4. Track
|
|
25
|
-
|
|
24
|
+
4. Track four steps in the current host plan: primary analysis, advisor launch, completion wait,
|
|
25
|
+
and verified synthesis.
|
|
26
26
|
|
|
27
27
|
## 1. Create the session and write the primary view
|
|
28
28
|
|
|
@@ -33,25 +33,11 @@ mkdir -p "${SESSION_PATH}"
|
|
|
33
33
|
```
|
|
34
34
|
|
|
35
35
|
Write Claude Code's evidence-backed analysis to `${SESSION_PATH}/claude-analysis.md` before
|
|
36
|
-
reading
|
|
36
|
+
reading the advisor output. Record the mode, checkout, scope, and evidence boundary.
|
|
37
37
|
|
|
38
|
-
## 2. Launch
|
|
38
|
+
## 2. Launch the advisor read-only
|
|
39
39
|
|
|
40
|
-
Give
|
|
41
|
-
Launch them concurrently when the host supports independent shell sessions; otherwise start both
|
|
42
|
-
before collecting either result.
|
|
43
|
-
|
|
44
|
-
Gemini:
|
|
45
|
-
|
|
46
|
-
```bash
|
|
47
|
-
gemini --approval-mode plan \
|
|
48
|
-
--output-format text \
|
|
49
|
-
-p "<bounded prompt>" \
|
|
50
|
-
> "${SESSION_PATH}/gemini-output.md" \
|
|
51
|
-
2> "${SESSION_PATH}/gemini-error.log"
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
Codex:
|
|
40
|
+
Give the advisor the bounded raw question and evidence without the primary conclusion.
|
|
55
41
|
|
|
56
42
|
```bash
|
|
57
43
|
codex exec -s read-only \
|
|
@@ -60,10 +46,9 @@ codex exec -s read-only \
|
|
|
60
46
|
2> "${SESSION_PATH}/codex-error.log"
|
|
61
47
|
```
|
|
62
48
|
|
|
63
|
-
Never enable
|
|
64
|
-
either advisor.
|
|
49
|
+
Never enable write-capable automation, danger-full-access, or permission bypass for the advisor.
|
|
65
50
|
|
|
66
|
-
## 3. Wait for completed
|
|
51
|
+
## 3. Wait for completed output
|
|
67
52
|
|
|
68
53
|
Run the bundled waiter in an asynchronous or yielded shell session and poll it at bounded
|
|
69
54
|
intervals; never hold one blocking tool call longer than 60 seconds.
|
|
@@ -73,28 +58,26 @@ python3 "$CLAUDE_PLUGIN_ROOT/skills/ultra-verify/scripts/verify_wait.py" \
|
|
|
73
58
|
"${SESSION_PATH}" --timeout 1200
|
|
74
59
|
```
|
|
75
60
|
|
|
76
|
-
The waiter reports
|
|
77
|
-
|
|
78
|
-
error log when it fails.
|
|
61
|
+
The waiter reports the advisor as `complete`, `failed`, `empty`, or `pending`. Read the output only
|
|
62
|
+
after its status is `complete` and the file is non-empty. Preserve the error log when it fails.
|
|
79
63
|
|
|
80
64
|
## 4. Verify and synthesize
|
|
81
65
|
|
|
82
|
-
Compare `claude-analysis.md` with
|
|
66
|
+
Compare `claude-analysis.md` with `codex-output.md`:
|
|
83
67
|
|
|
84
68
|
1. Verify claims against the current checkout, tests, runtime, or primary documentation.
|
|
85
|
-
2. Separate
|
|
86
|
-
3. Explain scope, version, and assumption differences before
|
|
69
|
+
2. Separate verified agreement, useful dissent, and unsupported assertions.
|
|
70
|
+
3. Explain scope, version, and assumption differences before judging the result.
|
|
87
71
|
4. Write `synthesis.md` and `metadata.json`, then return one Claude Code-owned conclusion.
|
|
88
72
|
|
|
89
|
-
Use `references/confidence-system.md` for
|
|
90
|
-
`references/cross-verify-modes.md` for mode-specific
|
|
91
|
-
|
|
73
|
+
Use `references/confidence-system.md` for evidence-based confidence and
|
|
74
|
+
`references/cross-verify-modes.md` for mode-specific expectations. Model agreement never overrides
|
|
75
|
+
failing tests or authoritative runtime evidence.
|
|
92
76
|
|
|
93
77
|
## Degraded operation
|
|
94
78
|
|
|
95
|
-
-
|
|
96
|
-
-
|
|
97
|
-
- Never block the underlying user task solely because an advisor is absent, unauthenticated, or
|
|
79
|
+
- Advisor failure: return the primary analysis with an explicit single-source warning.
|
|
80
|
+
- Never block the underlying user task solely because the advisor is absent, unauthenticated, or
|
|
98
81
|
slow.
|
|
99
82
|
|
|
100
83
|
## Session files
|
|
@@ -102,8 +85,6 @@ overrides failing tests or authoritative runtime evidence.
|
|
|
102
85
|
```text
|
|
103
86
|
.ultra/collab/<SESSION_ID>/
|
|
104
87
|
claude-analysis.md
|
|
105
|
-
gemini-output.md
|
|
106
|
-
gemini-error.log
|
|
107
88
|
codex-output.md
|
|
108
89
|
codex-error.log
|
|
109
90
|
metadata.json
|
|
@@ -4,47 +4,45 @@
|
|
|
4
4
|
{
|
|
5
5
|
"id": 1,
|
|
6
6
|
"prompt": "/ultra-verify decision 'Should we use PostgreSQL or MongoDB for our new microservice?'",
|
|
7
|
-
"expected_output": "Claude
|
|
7
|
+
"expected_output": "Claude writes its analysis first, launches Codex read-only, verifies both analyses, and returns one evidence-backed synthesis",
|
|
8
8
|
"assertions": [
|
|
9
|
-
"Claude writes its own analysis
|
|
10
|
-
"Launches
|
|
11
|
-
"Creates session directory under .ultra/collab/
|
|
12
|
-
"
|
|
13
|
-
"
|
|
14
|
-
"Includes per-AI recommendation summary and trade-off matrix"
|
|
9
|
+
"Claude writes its own analysis before reading advisor output",
|
|
10
|
+
"Launches Codex with a read-only sandbox",
|
|
11
|
+
"Creates a session directory under .ultra/collab/",
|
|
12
|
+
"Verifies consequential claims against authoritative evidence",
|
|
13
|
+
"Separates verified agreement, useful dissent, and unsupported assertions"
|
|
15
14
|
]
|
|
16
15
|
},
|
|
17
16
|
{
|
|
18
17
|
"id": 2,
|
|
19
|
-
"prompt": "/ultra-verify diagnose 'API returns 500 intermittently under load
|
|
20
|
-
"expected_output": "
|
|
18
|
+
"prompt": "/ultra-verify diagnose 'API returns 500 intermittently under load'",
|
|
19
|
+
"expected_output": "Independent hypotheses are reconciled through discriminating checks and evidence strength",
|
|
21
20
|
"assertions": [
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"Each hypothesis includes
|
|
25
|
-
"
|
|
21
|
+
"Both analyses provide ranked hypotheses",
|
|
22
|
+
"Duplicate hypotheses are merged by root cause",
|
|
23
|
+
"Each retained hypothesis includes a verification step",
|
|
24
|
+
"Investigation order follows evidence strength and verification cost"
|
|
26
25
|
]
|
|
27
26
|
},
|
|
28
27
|
{
|
|
29
28
|
"id": 3,
|
|
30
29
|
"prompt": "/ultra-verify audit",
|
|
31
|
-
"expected_output": "
|
|
30
|
+
"expected_output": "Findings are deduplicated and verified against the current checkout without vote-based severity",
|
|
32
31
|
"assertions": [
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
32
|
+
"Uses the same scope and severity model for both analyses",
|
|
33
|
+
"Verifies every reported location",
|
|
34
|
+
"Retains uniquely reported findings when source evidence confirms them",
|
|
35
|
+
"Severity does not depend on model agreement"
|
|
37
36
|
]
|
|
38
37
|
},
|
|
39
38
|
{
|
|
40
39
|
"id": 4,
|
|
41
|
-
"prompt": "/ultra-verify estimate 'Implement OAuth2 login with
|
|
42
|
-
"expected_output": "
|
|
40
|
+
"prompt": "/ultra-verify estimate 'Implement OAuth2 login with two providers'",
|
|
41
|
+
"expected_output": "Two estimates are reconciled by scope, assumptions, dependencies, and risk",
|
|
43
42
|
"assertions": [
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"Final recommended estimate with confidence level"
|
|
43
|
+
"Both analyses include work breakdowns and assumptions",
|
|
44
|
+
"Large differences are explained rather than averaged mechanically",
|
|
45
|
+
"Final range is supported by the reconciled work breakdown"
|
|
48
46
|
]
|
|
49
47
|
}
|
|
50
48
|
]
|
|
@@ -1,38 +1,21 @@
|
|
|
1
|
-
# Confidence
|
|
1
|
+
# Evidence-Based Confidence
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Cross-model agreement is a signal, not an authority. Assign confidence only after checking claims
|
|
4
|
+
against source code, tests, runtime output, or primary documentation.
|
|
4
5
|
|
|
5
6
|
## Levels
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
- **
|
|
10
|
-
|
|
8
|
+
- **Verified agreement**: both analyses agree and the consequential claims have authoritative
|
|
9
|
+
supporting evidence.
|
|
10
|
+
- **Resolved dissent**: the analyses differ, but current evidence supports one position and explains
|
|
11
|
+
the disagreement.
|
|
12
|
+
- **Unresolved**: available evidence cannot distinguish the positions or key assumptions remain
|
|
13
|
+
untested.
|
|
14
|
+
- **Single source**: the advisor failed or returned no usable output; no cross-model corroboration is
|
|
15
|
+
available.
|
|
11
16
|
|
|
12
|
-
|
|
13
|
-
- **Confidence**: High
|
|
14
|
-
- **Action**: Recommended — investigate the dissenting view for edge cases or alternative perspectives
|
|
15
|
-
- **Display**: `[MAJORITY 2/3]` + note which AI dissented and why
|
|
17
|
+
## Reporting
|
|
16
18
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
- **Display**: `[NO CONSENSUS]` + summary of each position
|
|
21
|
-
|
|
22
|
-
## Application by Mode
|
|
23
|
-
|
|
24
|
-
| Mode | Consensus | Majority | No Consensus |
|
|
25
|
-
|------|-----------|----------|--------------|
|
|
26
|
-
| decision | Strong recommendation | Recommended + dissent analysis | Present all options, suggest decomposition |
|
|
27
|
-
| diagnose | Investigate first | Investigate second | Check all, may need more data |
|
|
28
|
-
| audit | Critical — fix now | High — likely real | Investigate — may be false positive |
|
|
29
|
-
| estimate | Use average, high confidence | Investigate outlier | Decompose task further |
|
|
30
|
-
|
|
31
|
-
## Degraded Confidence
|
|
32
|
-
|
|
33
|
-
When only 2 AIs respond (one failed):
|
|
34
|
-
|
|
35
|
-
- **2/2 Agree**: Equivalent to Majority (not Consensus — missing perspective)
|
|
36
|
-
- **2/2 Differ**: Low confidence — note missing third opinion
|
|
37
|
-
|
|
38
|
-
When only 1 AI responds: Claude-only analysis — explicitly mark as single-source, no consensus scoring.
|
|
19
|
+
Record which claims were verified, the evidence used, useful dissent, unsupported assertions, and
|
|
20
|
+
remaining uncertainty. Model agreement never upgrades a claim contradicted by tests or runtime
|
|
21
|
+
evidence, and a uniquely reported finding remains valid when independently verified.
|
|
@@ -1,73 +1,28 @@
|
|
|
1
1
|
# Ultra Verify Modes
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
All four modes compare an independently written primary analysis with one read-only external
|
|
4
|
+
advisor. Agreement is useful corroboration, not proof; evidence decides the final result.
|
|
4
5
|
|
|
5
6
|
## 1. Decision (`decision`)
|
|
6
7
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
1. Present the decision context, constraints, and options to all three AIs independently
|
|
11
|
-
2. Each AI provides: recommendation, pros/cons, risks, trade-offs
|
|
12
|
-
3. Synthesis ranks options by consensus:
|
|
13
|
-
- 3/3 recommend same → **Strong recommendation**
|
|
14
|
-
- 2/3 agree → **Recommended** with dissenting view analyzed
|
|
15
|
-
- All different → Present all three with trade-off matrix
|
|
16
|
-
|
|
17
|
-
**Output structure:**
|
|
18
|
-
- Per-AI recommendation summary
|
|
19
|
-
- Consensus matrix (which AI recommends what)
|
|
20
|
-
- Trade-off analysis
|
|
21
|
-
- Final recommendation with confidence level
|
|
8
|
+
Give both analyses the same decision context, constraints, and options. Compare recommendation,
|
|
9
|
+
trade-offs, risks, and assumptions. When they differ, identify the assumption or evidence that
|
|
10
|
+
caused the split and resolve it before recommending an option.
|
|
22
11
|
|
|
23
12
|
## 2. Diagnose (`diagnose`)
|
|
24
13
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
1. Provide symptoms, error messages, relevant code, and recent changes to all three AIs
|
|
29
|
-
2. Each AI provides its top-3 hypotheses with evidence and verification steps
|
|
30
|
-
3. Synthesis:
|
|
31
|
-
- Hypotheses appearing in 3/3 lists → **Most likely** (investigate first)
|
|
32
|
-
- Hypotheses in 2/3 lists → **Probable** (investigate second)
|
|
33
|
-
- Unique hypotheses → **Worth checking** (may catch edge cases)
|
|
34
|
-
|
|
35
|
-
**Output structure:**
|
|
36
|
-
- Ranked hypothesis list (by consensus count)
|
|
37
|
-
- Per-hypothesis: description, evidence, verification steps, suggested fix
|
|
38
|
-
- Recommended investigation order
|
|
14
|
+
Provide symptoms, exact errors, relevant code, and recent changes. Each analysis should rank its
|
|
15
|
+
root-cause hypotheses and provide discriminating checks. Merge duplicate hypotheses, retain useful
|
|
16
|
+
unique ones, and order investigation by evidence strength and verification cost.
|
|
39
17
|
|
|
40
18
|
## 3. Audit (`audit`)
|
|
41
19
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
1. Provide the code scope to all three AIs with the same audit prompt
|
|
46
|
-
2. Each AI reports findings with severity and location
|
|
47
|
-
3. Synthesis grades findings by consensus:
|
|
48
|
-
- Found by 3/3 → **Critical** — fix immediately
|
|
49
|
-
- Found by 2/3 → **High** — likely real, investigate
|
|
50
|
-
- Found by 1/3 → **Investigate** — may be false positive or edge case
|
|
51
|
-
|
|
52
|
-
**Output structure:**
|
|
53
|
-
- Findings table: description, severity, consensus count, which AIs found it
|
|
54
|
-
- Grouped by consensus level (Critical → High → Investigate)
|
|
55
|
-
- Action items prioritized by consensus
|
|
20
|
+
Use the same code scope and severity model for both analyses. Deduplicate findings by root cause,
|
|
21
|
+
verify every location against the current checkout, and retain a single-model finding when source
|
|
22
|
+
evidence confirms it. Never derive severity from model agreement alone.
|
|
56
23
|
|
|
57
24
|
## 4. Estimate (`estimate`)
|
|
58
25
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
1. Describe the task, requirements, and constraints to all three AIs
|
|
63
|
-
2. Each AI provides: estimate (time/effort), breakdown, assumptions, risks
|
|
64
|
-
3. Synthesis evaluates convergence:
|
|
65
|
-
- All within 20% of each other → **High confidence** estimate (use average)
|
|
66
|
-
- One outlier → **Moderate confidence** — investigate the outlier's reasoning
|
|
67
|
-
- All significantly different → **Low confidence** — task needs decomposition
|
|
68
|
-
|
|
69
|
-
**Output structure:**
|
|
70
|
-
- Per-AI estimate with breakdown and assumptions
|
|
71
|
-
- Convergence analysis
|
|
72
|
-
- Recommended estimate with confidence level
|
|
73
|
-
- Risk factors that could shift the estimate
|
|
26
|
+
Require a breakdown, assumptions, dependencies, and risk range from both analyses. Explain large
|
|
27
|
+
differences by locating mismatched scope or assumptions. Return a range supported by the reconciled
|
|
28
|
+
work breakdown rather than averaging estimates mechanically.
|