ultra-builder-pro-cli 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +31 -6
  2. package/README.md +14 -16
  3. package/adapters/_shared/codex-assets.cjs +32 -151
  4. package/adapters/_shared/frontmatter.cjs +2 -2
  5. package/adapters/_shared/handbook.cjs +3 -3
  6. package/adapters/_shared/path-rewrite.cjs +0 -3
  7. package/adapters/_shared/runtime-assets.cjs +11 -4
  8. package/adapters/_shared/tests/path-rewrite.test.cjs +1 -3
  9. package/adapters/_shared/tests/resolve-target.test.cjs +1 -11
  10. package/adapters/_shared/tests/runtime-assets.test.cjs +5 -6
  11. package/adapters/opencode.js +1 -6
  12. package/adapters/tests/claude.test.cjs +1 -4
  13. package/adapters/tests/codex.test.cjs +5 -6
  14. package/adapters/tests/opencode.test.cjs +1 -2
  15. package/bin/install.js +6 -9
  16. package/bin/orchestrator.js +11 -4
  17. package/docs/AGENT-CONTEXT.md +4 -5
  18. package/docs/ARCHITECTURE.md +3 -4
  19. package/docs/LEGACY-HERMES.md +5 -81
  20. package/docs/PLAN.zh-CN.md +35 -42
  21. package/docs/ROADMAP.md +3 -3
  22. package/docs/RUNTIME-COMPAT-MATRIX.md +61 -60
  23. package/mcp-server/lib/pricing.cjs +0 -6
  24. package/mcp-server/lib/state-ops.cjs +7 -0
  25. package/mcp-server/lib/telemetry.cjs +4 -0
  26. package/mcp-server/tests/server-contract.test.cjs +17 -0
  27. package/mcp-server/tests/state-ops.test.cjs +23 -0
  28. package/mcp-server/tests/telemetry.test.cjs +14 -0
  29. package/orchestrator/dispatch-rules.cjs +15 -4
  30. package/orchestrator/tests/daemon.test.cjs +13 -12
  31. package/orchestrator/tests/dispatch-rules.test.cjs +12 -10
  32. package/package.json +3 -5
  33. package/skills/ultra-init/SKILL.md +0 -1
  34. package/skills/ultra-verify/SKILL.md +22 -41
  35. package/skills/ultra-verify/evals/evals.json +22 -24
  36. package/skills/ultra-verify/references/confidence-system.md +15 -32
  37. package/skills/ultra-verify/references/cross-verify-modes.md +14 -59
  38. package/skills/ultra-verify/scripts/verify_wait.py +44 -81
  39. package/spec/mcp-tools.yaml +3 -3
  40. package/spec/schemas/skill-manifest.schema.json +2 -2
  41. package/spec/schemas/state-db.sql +2 -2
  42. package/adapters/_shared/md-to-toml.cjs +0 -54
  43. package/adapters/_shared/tests/md-to-toml.test.cjs +0 -48
  44. package/adapters/gemini.js +0 -204
  45. package/adapters/tests/gemini.test.cjs +0 -112
  46. package/skills/gemini-collab/SKILL.md +0 -51
  47. package/skills/gemini-collab/evals/evals.json +0 -42
@@ -13,9 +13,11 @@
13
13
  // parallel orchestrator (8B.2) gets a richer decision surface: wave state,
14
14
  // deps readiness, and custom rule injection.
15
15
 
16
+ const { isSupportedRuntime } = require('../adapters/_shared/runtime-assets.cjs');
17
+
16
18
  const ROUTE_PREFERENCES = Object.freeze({
17
- haiku: ['opencode', 'gemini', 'claude', 'codex'],
18
- sonnet: ['claude', 'codex', 'opencode', 'gemini'],
19
+ haiku: ['opencode', 'claude', 'codex'],
20
+ sonnet: ['claude', 'codex', 'opencode'],
19
21
  opus: ['claude', 'codex'],
20
22
  });
21
23
 
@@ -73,12 +75,21 @@ const DEFAULT_RULES = Object.freeze([
73
75
  ]);
74
76
 
75
77
  function evaluate(ctx, rules = DEFAULT_RULES) {
78
+ const normalizedCtx = {
79
+ ...ctx,
80
+ available_runtimes: Array.isArray(ctx && ctx.available_runtimes)
81
+ ? [...new Set(ctx.available_runtimes.filter(isSupportedRuntime))]
82
+ : [],
83
+ };
76
84
  const sorted = [...rules].sort((a, b) => (b.priority || 0) - (a.priority || 0));
77
85
  for (const rule of sorted) {
78
- if (rule.when(ctx)) {
86
+ if (rule.when(normalizedCtx)) {
79
87
  let runtime = null;
80
88
  if (rule.action === 'spawn_agent') {
81
- runtime = rule.resolve ? rule.resolve(ctx) : (rule.runtime || null);
89
+ runtime = rule.resolve ? rule.resolve(normalizedCtx) : (rule.runtime || null);
90
+ if (!isSupportedRuntime(runtime)) {
91
+ return { rule_id: rule.id, action: 'block', runtime: null };
92
+ }
82
93
  }
83
94
  return { rule_id: rule.id, action: rule.action, runtime };
84
95
  }
@@ -15,6 +15,7 @@ const { execFileSync } = require('node:child_process');
15
15
  const { initStateDb, closeStateDb } = require('../../mcp-server/lib/state-db.cjs');
16
16
  const ops = require('../../mcp-server/lib/state-ops.cjs');
17
17
  const daemon = require('../daemon.cjs');
18
+ const RETIRED_RUNTIME = ['gem', 'ini'].join('');
18
19
 
19
20
  const LONG_SLEEP_CMD = process.execPath;
20
21
  const LONG_SLEEP_ARGS = ['-e', 'setInterval(() => {}, 60000);'];
@@ -55,15 +56,15 @@ function cleanup(repoRoot, db, handle) {
55
56
  test('routeTask: haiku hint prefers cheap runtimes', () => {
56
57
  const runtime = daemon.routeTask(
57
58
  { id: 't', complexity_hint: 'haiku' },
58
- ['claude', 'opencode', 'codex', 'gemini'],
59
+ ['claude', 'opencode', 'codex'],
59
60
  );
60
- assert.ok(['opencode', 'gemini'].includes(runtime), `got ${runtime}`);
61
+ assert.equal(runtime, 'opencode');
61
62
  });
62
63
 
63
64
  test('routeTask: opus hint prefers strong runtimes', () => {
64
65
  const runtime = daemon.routeTask(
65
66
  { id: 't', complexity_hint: 'opus' },
66
- ['claude', 'opencode', 'codex', 'gemini'],
67
+ ['claude', 'opencode', 'codex'],
67
68
  );
68
69
  assert.ok(['claude', 'codex'].includes(runtime), `got ${runtime}`);
69
70
  });
@@ -71,23 +72,23 @@ test('routeTask: opus hint prefers strong runtimes', () => {
71
72
  test('routeTask: sonnet hint middle-ground', () => {
72
73
  const runtime = daemon.routeTask(
73
74
  { id: 't', complexity_hint: 'sonnet' },
74
- ['claude', 'opencode', 'codex', 'gemini'],
75
+ ['claude', 'opencode', 'codex'],
75
76
  );
76
- assert.ok(['claude', 'codex', 'opencode', 'gemini'].includes(runtime));
77
+ assert.ok(['claude', 'codex', 'opencode'].includes(runtime));
77
78
  });
78
79
 
79
80
  test('routeTask: respects availability constraint', () => {
80
81
  const runtime = daemon.routeTask(
81
82
  { id: 't', complexity_hint: 'opus' },
82
- ['opencode', 'gemini'],
83
+ ['opencode'],
83
84
  );
84
85
  // Prefers claude/codex but those aren't available → fall back to any.
85
- assert.ok(['opencode', 'gemini'].includes(runtime));
86
+ assert.equal(runtime, 'opencode');
86
87
  });
87
88
 
88
- test('routeTask: no hint → first available', () => {
89
- const runtime = daemon.routeTask({ id: 't' }, ['gemini', 'claude']);
90
- assert.ok(['gemini', 'claude'].includes(runtime));
89
+ test('routeTask: no hint skips unsupported runtimes', () => {
90
+ const runtime = daemon.routeTask({ id: 't' }, [RETIRED_RUNTIME, 'claude']);
91
+ assert.equal(runtime, 'claude');
91
92
  });
92
93
 
93
94
  test('routeTask: no runtimes available → null', () => {
@@ -244,7 +245,7 @@ test('runDaemon applies complexity_hint route', async () => {
244
245
  });
245
246
  handle = daemon.runDaemon({
246
247
  db, repoRoot,
247
- runtimes: ['claude', 'opencode', 'gemini'],
248
+ runtimes: ['claude', 'opencode'],
248
249
  pollMs: 50,
249
250
  command: LONG_SLEEP_CMD,
250
251
  commandArgs: LONG_SLEEP_ARGS,
@@ -252,7 +253,7 @@ test('runDaemon applies complexity_hint route', async () => {
252
253
  await new Promise((r) => setTimeout(r, 300));
253
254
  const sessions = db.prepare("SELECT * FROM sessions WHERE task_id = 'd-opus'").all();
254
255
  assert.equal(sessions.length, 1);
255
- // opus should prefer claude over opencode/gemini.
256
+ // opus should prefer claude over opencode.
256
257
  assert.equal(sessions[0].runtime, 'claude');
257
258
  } finally {
258
259
  cleanup(repoRoot, db, handle);
@@ -11,12 +11,13 @@ const { test } = require('node:test');
11
11
  const assert = require('node:assert/strict');
12
12
 
13
13
  const { evaluate, DEFAULT_RULES, ROUTE_PREFERENCES } = require('../dispatch-rules.cjs');
14
+ const RETIRED_RUNTIME = ['gem', 'ini'].join('');
14
15
 
15
16
  function baseCtx(overrides = {}) {
16
17
  return {
17
18
  task: { id: 't1' },
18
19
  deps_ready: true,
19
- available_runtimes: ['claude', 'opencode', 'codex', 'gemini'],
20
+ available_runtimes: ['claude', 'opencode', 'codex'],
20
21
  breaker_state: 'ok',
21
22
  wave: null,
22
23
  ...overrides,
@@ -42,7 +43,7 @@ test('evaluate: opus hint + only codex available → codex', () => {
42
43
  test('evaluate: opus hint + preference exhausted → first available fallback', () => {
43
44
  const d = evaluate(baseCtx({
44
45
  task: { id: 't', complexity_hint: 'opus' },
45
- available_runtimes: ['opencode', 'gemini'],
46
+ available_runtimes: ['opencode'],
46
47
  }));
47
48
  assert.equal(d.action, 'spawn_agent');
48
49
  assert.equal(d.runtime, 'opencode');
@@ -58,13 +59,13 @@ test('evaluate: sonnet hint + all available → claude', () => {
58
59
  assert.equal(d.runtime, 'claude');
59
60
  });
60
61
 
61
- test('evaluate: no hint → first available fallback', () => {
62
+ test('evaluate: retired runtimes are discarded before fallback routing', () => {
62
63
  const d = evaluate(baseCtx({
63
64
  task: { id: 't' },
64
- available_runtimes: ['gemini', 'claude'],
65
+ available_runtimes: [RETIRED_RUNTIME, 'claude'],
65
66
  }));
66
67
  assert.equal(d.action, 'spawn_agent');
67
- assert.equal(d.runtime, 'gemini');
68
+ assert.equal(d.runtime, 'claude');
68
69
  });
69
70
 
70
71
  // ─── block / defer paths (new in 8B.1) ────────────────────────────────────
@@ -97,20 +98,21 @@ test('evaluate: serial wave already has running task → defer', () => {
97
98
 
98
99
  // ─── composability: custom rules override defaults ────────────────────────
99
100
 
100
- test('evaluate: custom high-priority rule overrides defaults', () => {
101
+ test('evaluate: custom rules cannot select an unsupported runtime', () => {
101
102
  const customRules = [
102
103
  ...DEFAULT_RULES,
103
104
  {
104
- id: 'force-gemini',
105
+ id: 'force-retired-runtime',
105
106
  priority: 999,
106
107
  when: () => true,
107
108
  action: 'spawn_agent',
108
- resolve: () => 'gemini',
109
+ resolve: () => RETIRED_RUNTIME,
109
110
  },
110
111
  ];
111
112
  const d = evaluate(baseCtx({ task: { id: 't', complexity_hint: 'opus' } }), customRules);
112
- assert.equal(d.runtime, 'gemini');
113
- assert.equal(d.rule_id, 'force-gemini');
113
+ assert.equal(d.action, 'block');
114
+ assert.equal(d.runtime, null);
115
+ assert.equal(d.rule_id, 'force-retired-runtime');
114
116
  });
115
117
 
116
118
  // ─── ROUTE_PREFERENCES shape guard (regression: don't drift from Phase 5.4) ─
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "ultra-builder-pro-cli",
3
- "version": "0.5.3",
4
- "description": "Multi-runtime installer for the Ultra Builder Pro agent engineering system — deploys commands, agents, skills, hooks, and MCP server to Claude Code, OpenCode, Codex CLI, and Gemini CLI.",
3
+ "version": "0.6.0",
4
+ "description": "Multi-runtime installer for the Ultra Builder Pro agent engineering system — deploys commands, agents, skills, hooks, and MCP server to Claude Code, OpenCode, and Codex CLI.",
5
5
  "bin": {
6
6
  "ultra-builder-pro-cli": "bin/install.js",
7
7
  "ubp": "bin/install.js",
@@ -33,7 +33,6 @@
33
33
  "skills/testing-rules",
34
34
  "skills/cc-collab",
35
35
  "skills/codex-collab",
36
- "skills/gemini-collab",
37
36
  "skills/ultra-verify",
38
37
  "hooks/*.py",
39
38
  "hooks/adapters/*.py",
@@ -60,7 +59,6 @@
60
59
  "claude-code",
61
60
  "opencode",
62
61
  "codex",
63
- "gemini",
64
62
  "agent",
65
63
  "cli",
66
64
  "multi-runtime",
@@ -87,7 +85,7 @@
87
85
  "test:state": "node --test 'mcp-server/**/*.test.cjs'",
88
86
  "test:orch": "node --test 'orchestrator/**/*.test.cjs'",
89
87
  "test:spec": "node spec/scripts/test-all.cjs",
90
- "test:rest": "node --test adapters/_shared/tests/*.test.cjs adapters/tests/*.test.cjs ultra-tools/*.test.cjs tests/conformance/**/*.test.cjs tests/install.test.cjs tests/package-smoke.test.cjs",
88
+ "test:rest": "node --test adapters/_shared/tests/*.test.cjs adapters/tests/*.test.cjs ultra-tools/*.test.cjs tests/conformance/**/*.test.cjs tests/install.test.cjs tests/package-smoke.test.cjs tests/retired-runtime.test.cjs",
91
89
  "test:all": "npm run test:state && npm run test:orch && npm run test:spec && npm run test:rest",
92
90
  "test:hooks": "pytest hooks/tests -q",
93
91
  "verify:release": "npm run test:all && npm run test:hooks && npm audit --omit=dev --audit-level=high",
@@ -133,7 +133,6 @@ CLI 最后一行是 `{ "ok": true, "data": { ... } }`(见 spec/cli-protocol.md
133
133
  | Claude | `/ultra-init [name] [type] [stack] [git]` — 命令薄壳拉起此 skill |
134
134
  | OpenCode | `/ultra-init …` 同上(agent frontmatter 小写化) |
135
135
  | Codex | `codex exec "run /ultra-init …"`(Phase 4.4 adapter 接) |
136
- | Gemini | `gemini --prompt "run /ultra-init …"`(Phase 4.5 extension 接) |
137
136
 
138
137
  ## 输出锚点
139
138
 
@@ -1,13 +1,13 @@
1
1
  ---
2
2
  name: ultra-verify
3
- description: Run three-way verification with the current host as primary and read-only Gemini and Codex advisors, then verify and synthesize the evidence.
3
+ description: Run independent two-model verification with the current host as primary and a read-only Codex advisor, then verify and synthesize the evidence.
4
4
  ---
5
5
 
6
- # Ultra Verify — Three-Way Verification
6
+ # Ultra Verify — Independent Two-Model Verification
7
7
 
8
8
  Claude Code remains primary. It writes the first independent analysis, verifies consequential
9
- claims, and owns the final answer. Gemini and Codex are untrusted read-only advisors. Use this
10
- workflow only when the user explicitly asks for cross-model or three-way verification.
9
+ claims, and owns the final answer. Codex is an untrusted read-only advisor. Use this workflow only
10
+ when the user explicitly asks for cross-model verification.
11
11
 
12
12
  ## Modes
13
13
 
@@ -18,11 +18,11 @@ workflow only when the user explicitly asks for cross-model or three-way verific
18
18
 
19
19
  ## Preconditions
20
20
 
21
- 1. Confirm `gemini --version` and `codex --version` plus authentication.
21
+ 1. Confirm `codex --version` plus authentication.
22
22
  2. Define the exact workspace, question, evidence standard, and response shape.
23
23
  3. Do not send secrets, unrelated files, or an unbounded home directory.
24
- 4. Track the four workflow steps in the current host plan: primary analysis, advisor launch,
25
- completion wait, and verified synthesis.
24
+ 4. Track four steps in the current host plan: primary analysis, advisor launch, completion wait,
25
+ and verified synthesis.
26
26
 
27
27
  ## 1. Create the session and write the primary view
28
28
 
@@ -33,25 +33,11 @@ mkdir -p "${SESSION_PATH}"
33
33
  ```
34
34
 
35
35
  Write Claude Code's evidence-backed analysis to `${SESSION_PATH}/claude-analysis.md` before
36
- reading either advisor. Record the mode, checkout, scope, and evidence boundary.
36
+ reading the advisor output. Record the mode, checkout, scope, and evidence boundary.
37
37
 
38
- ## 2. Launch both advisors read-only
38
+ ## 2. Launch the advisor read-only
39
39
 
40
- Give both advisors the same bounded raw question and evidence, without the primary conclusion.
41
- Launch them concurrently when the host supports independent shell sessions; otherwise start both
42
- before collecting either result.
43
-
44
- Gemini:
45
-
46
- ```bash
47
- gemini --approval-mode plan \
48
- --output-format text \
49
- -p "<bounded prompt>" \
50
- > "${SESSION_PATH}/gemini-output.md" \
51
- 2> "${SESSION_PATH}/gemini-error.log"
52
- ```
53
-
54
- Codex:
40
+ Give the advisor the bounded raw question and evidence without the primary conclusion.
55
41
 
56
42
  ```bash
57
43
  codex exec -s read-only \
@@ -60,10 +46,9 @@ codex exec -s read-only \
60
46
  2> "${SESSION_PATH}/codex-error.log"
61
47
  ```
62
48
 
63
- Never enable auto-edit, write-capable automation, danger-full-access, or permission bypass for
64
- either advisor.
49
+ Never enable write-capable automation, danger-full-access, or permission bypass for the advisor.
65
50
 
66
- ## 3. Wait for completed outputs
51
+ ## 3. Wait for completed output
67
52
 
68
53
  Run the bundled waiter in an asynchronous or yielded shell session and poll it at bounded
69
54
  intervals; never hold one blocking tool call longer than 60 seconds.
@@ -73,28 +58,26 @@ python3 "$CLAUDE_PLUGIN_ROOT/skills/ultra-verify/scripts/verify_wait.py" \
73
58
  "${SESSION_PATH}" --timeout 1200
74
59
  ```
75
60
 
76
- The waiter reports each advisor as `complete`, `failed`, `empty`, or `pending`. Read an advisor
77
- output only after the corresponding status is `complete` and the file is non-empty. Preserve its
78
- error log when it fails.
61
+ The waiter reports the advisor as `complete`, `failed`, `empty`, or `pending`. Read the output only
62
+ after its status is `complete` and the file is non-empty. Preserve the error log when it fails.
79
63
 
80
64
  ## 4. Verify and synthesize
81
65
 
82
- Compare `claude-analysis.md` with each completed advisor output:
66
+ Compare `claude-analysis.md` with `codex-output.md`:
83
67
 
84
68
  1. Verify claims against the current checkout, tests, runtime, or primary documentation.
85
- 2. Separate consensus, majority views, useful dissent, and unsupported assertions.
86
- 3. Explain scope, version, and assumption differences before scoring agreement.
69
+ 2. Separate verified agreement, useful dissent, and unsupported assertions.
70
+ 3. Explain scope, version, and assumption differences before judging the result.
87
71
  4. Write `synthesis.md` and `metadata.json`, then return one Claude Code-owned conclusion.
88
72
 
89
- Use `references/confidence-system.md` for scoring and
90
- `references/cross-verify-modes.md` for mode-specific evidence expectations. Model agreement never
91
- overrides failing tests or authoritative runtime evidence.
73
+ Use `references/confidence-system.md` for evidence-based confidence and
74
+ `references/cross-verify-modes.md` for mode-specific expectations. Model agreement never overrides
75
+ failing tests or authoritative runtime evidence.
92
76
 
93
77
  ## Degraded operation
94
78
 
95
- - One advisor fails: continue with the primary view plus the available advisor and name the gap.
96
- - Both advisors fail: return primary-only analysis with an explicit single-source warning.
97
- - Never block the underlying user task solely because an advisor is absent, unauthenticated, or
79
+ - Advisor failure: return the primary analysis with an explicit single-source warning.
80
+ - Never block the underlying user task solely because the advisor is absent, unauthenticated, or
98
81
  slow.
99
82
 
100
83
  ## Session files
@@ -102,8 +85,6 @@ overrides failing tests or authoritative runtime evidence.
102
85
  ```text
103
86
  .ultra/collab/<SESSION_ID>/
104
87
  claude-analysis.md
105
- gemini-output.md
106
- gemini-error.log
107
88
  codex-output.md
108
89
  codex-error.log
109
90
  metadata.json
@@ -4,47 +4,45 @@
4
4
  {
5
5
  "id": 1,
6
6
  "prompt": "/ultra-verify decision 'Should we use PostgreSQL or MongoDB for our new microservice?'",
7
- "expected_output": "Claude answers first, then launches Gemini and Codex in parallel, collects all three analyses, and presents a synthesis with consensus scoring",
7
+ "expected_output": "Claude writes its analysis first, launches Codex read-only, verifies both analyses, and returns one evidence-backed synthesis",
8
8
  "assertions": [
9
- "Claude writes its own analysis BEFORE reading external AI output",
10
- "Launches Gemini and Codex in parallel (run_in_background)",
11
- "Creates session directory under .ultra/collab/ with verify-decision in name",
12
- "Reads all three output files with Read tool",
13
- "Presents confidence level (consensus/majority/no consensus)",
14
- "Includes per-AI recommendation summary and trade-off matrix"
9
+ "Claude writes its own analysis before reading advisor output",
10
+ "Launches Codex with a read-only sandbox",
11
+ "Creates a session directory under .ultra/collab/",
12
+ "Verifies consequential claims against authoritative evidence",
13
+ "Separates verified agreement, useful dissent, and unsupported assertions"
15
14
  ]
16
15
  },
17
16
  {
18
17
  "id": 2,
19
- "prompt": "/ultra-verify diagnose 'API returns 500 intermittently under load, logs show connection pool exhaustion'",
20
- "expected_output": "Three sets of hypotheses collected and ranked by consensus count",
18
+ "prompt": "/ultra-verify diagnose 'API returns 500 intermittently under load'",
19
+ "expected_output": "Independent hypotheses are reconciled through discriminating checks and evidence strength",
21
20
  "assertions": [
22
- "Each AI provides top-3 hypotheses",
23
- "Hypotheses ranked by consensus count (3/3 > 2/3 > 1/3)",
24
- "Each hypothesis includes evidence and verification steps",
25
- "Recommended investigation order based on consensus"
21
+ "Both analyses provide ranked hypotheses",
22
+ "Duplicate hypotheses are merged by root cause",
23
+ "Each retained hypothesis includes a verification step",
24
+ "Investigation order follows evidence strength and verification cost"
26
25
  ]
27
26
  },
28
27
  {
29
28
  "id": 3,
30
29
  "prompt": "/ultra-verify audit",
31
- "expected_output": "Three-way code audit with findings graded by consensus (3=critical, 2=high, 1=investigate)",
30
+ "expected_output": "Findings are deduplicated and verified against the current checkout without vote-based severity",
32
31
  "assertions": [
33
- "Sends same audit prompt to all three AIs",
34
- "Findings graded by how many AIs found them",
35
- "3/3 findings marked Critical, 2/3 marked High, 1/3 marked Investigate",
36
- "Action items prioritized by consensus count"
32
+ "Uses the same scope and severity model for both analyses",
33
+ "Verifies every reported location",
34
+ "Retains uniquely reported findings when source evidence confirms them",
35
+ "Severity does not depend on model agreement"
37
36
  ]
38
37
  },
39
38
  {
40
39
  "id": 4,
41
- "prompt": "/ultra-verify estimate 'Implement OAuth2 login with Google and GitHub providers'",
42
- "expected_output": "Three effort estimates compared for convergence",
40
+ "prompt": "/ultra-verify estimate 'Implement OAuth2 login with two providers'",
41
+ "expected_output": "Two estimates are reconciled by scope, assumptions, dependencies, and risk",
43
42
  "assertions": [
44
- "Each AI provides estimate with breakdown and assumptions",
45
- "Convergence analysis (within 20% = high confidence)",
46
- "Outlier estimates investigated",
47
- "Final recommended estimate with confidence level"
43
+ "Both analyses include work breakdowns and assumptions",
44
+ "Large differences are explained rather than averaged mechanically",
45
+ "Final range is supported by the reconciled work breakdown"
48
46
  ]
49
47
  }
50
48
  ]
@@ -1,38 +1,21 @@
1
- # Confidence System
1
+ # Evidence-Based Confidence
2
2
 
3
- Confidence scoring based on AI consensus for cross-verify results.
3
+ Cross-model agreement is a signal, not an authority. Assign confidence only after checking claims
4
+ against source code, tests, runtime output, or primary documentation.
4
5
 
5
6
  ## Levels
6
7
 
7
- ### Consensus (3/3 Agree)
8
- - **Confidence**: Highest
9
- - **Action**: Strongly recommended — proceed with high confidence
10
- - **Display**: `[CONSENSUS 3/3]`
8
+ - **Verified agreement**: both analyses agree and the consequential claims have authoritative
9
+ supporting evidence.
10
+ - **Resolved dissent**: the analyses differ, but current evidence supports one position and explains
11
+ the disagreement.
12
+ - **Unresolved**: available evidence cannot distinguish the positions or key assumptions remain
13
+ untested.
14
+ - **Single source**: the advisor failed or returned no usable output; no cross-model corroboration is
15
+ available.
11
16
 
12
- ### Majority (2/3 Agree)
13
- - **Confidence**: High
14
- - **Action**: Recommended — investigate the dissenting view for edge cases or alternative perspectives
15
- - **Display**: `[MAJORITY 2/3]` + note which AI dissented and why
17
+ ## Reporting
16
18
 
17
- ### No Consensus (All Differ)
18
- - **Confidence**: Low
19
- - **Action**: Decompose the problem into smaller questions, gather more data, or accept that the answer is genuinely ambiguous
20
- - **Display**: `[NO CONSENSUS]` + summary of each position
21
-
22
- ## Application by Mode
23
-
24
- | Mode | Consensus | Majority | No Consensus |
25
- |------|-----------|----------|--------------|
26
- | decision | Strong recommendation | Recommended + dissent analysis | Present all options, suggest decomposition |
27
- | diagnose | Investigate first | Investigate second | Check all, may need more data |
28
- | audit | Critical — fix now | High — likely real | Investigate — may be false positive |
29
- | estimate | Use average, high confidence | Investigate outlier | Decompose task further |
30
-
31
- ## Degraded Confidence
32
-
33
- When only 2 AIs respond (one failed):
34
-
35
- - **2/2 Agree**: Equivalent to Majority (not Consensus — missing perspective)
36
- - **2/2 Differ**: Low confidence — note missing third opinion
37
-
38
- When only 1 AI responds: Claude-only analysis — explicitly mark as single-source, no consensus scoring.
19
+ Record which claims were verified, the evidence used, useful dissent, unsupported assertions, and
20
+ remaining uncertainty. Model agreement never upgrades a claim contradicted by tests or runtime
21
+ evidence, and a uniquely reported finding remains valid when independently verified.
@@ -1,73 +1,28 @@
1
1
  # Ultra Verify Modes
2
2
 
3
- Four modes for three-way AI verification. All modes follow the safe orchestration contract in the parent `SKILL.md`.
3
+ All four modes compare an independently written primary analysis with one read-only external
4
+ advisor. Agreement is useful corroboration, not proof; evidence decides the final result.
4
5
 
5
6
  ## 1. Decision (`decision`)
6
7
 
7
- For architecture decisions, technology choices, or design trade-offs.
8
-
9
- **Flow:**
10
- 1. Present the decision context, constraints, and options to all three AIs independently
11
- 2. Each AI provides: recommendation, pros/cons, risks, trade-offs
12
- 3. Synthesis ranks options by consensus:
13
- - 3/3 recommend same → **Strong recommendation**
14
- - 2/3 agree → **Recommended** with dissenting view analyzed
15
- - All different → Present all three with trade-off matrix
16
-
17
- **Output structure:**
18
- - Per-AI recommendation summary
19
- - Consensus matrix (which AI recommends what)
20
- - Trade-off analysis
21
- - Final recommendation with confidence level
8
+ Give both analyses the same decision context, constraints, and options. Compare recommendation,
9
+ trade-offs, risks, and assumptions. When they differ, identify the assumption or evidence that
10
+ caused the split and resolve it before recommending an option.
22
11
 
23
12
  ## 2. Diagnose (`diagnose`)
24
13
 
25
- For bug diagnosis, root cause analysis, or troubleshooting.
26
-
27
- **Flow:**
28
- 1. Provide symptoms, error messages, relevant code, and recent changes to all three AIs
29
- 2. Each AI provides its top-3 hypotheses with evidence and verification steps
30
- 3. Synthesis:
31
- - Hypotheses appearing in 3/3 lists → **Most likely** (investigate first)
32
- - Hypotheses in 2/3 lists → **Probable** (investigate second)
33
- - Unique hypotheses → **Worth checking** (may catch edge cases)
34
-
35
- **Output structure:**
36
- - Ranked hypothesis list (by consensus count)
37
- - Per-hypothesis: description, evidence, verification steps, suggested fix
38
- - Recommended investigation order
14
+ Provide symptoms, exact errors, relevant code, and recent changes. Each analysis should rank its
15
+ root-cause hypotheses and provide discriminating checks. Merge duplicate hypotheses, retain useful
16
+ unique ones, and order investigation by evidence strength and verification cost.
39
17
 
40
18
  ## 3. Audit (`audit`)
41
19
 
42
- For code review, security audit, or quality assessment.
43
-
44
- **Flow:**
45
- 1. Provide the code scope to all three AIs with the same audit prompt
46
- 2. Each AI reports findings with severity and location
47
- 3. Synthesis grades findings by consensus:
48
- - Found by 3/3 → **Critical** — fix immediately
49
- - Found by 2/3 → **High** — likely real, investigate
50
- - Found by 1/3 → **Investigate** — may be false positive or edge case
51
-
52
- **Output structure:**
53
- - Findings table: description, severity, consensus count, which AIs found it
54
- - Grouped by consensus level (Critical → High → Investigate)
55
- - Action items prioritized by consensus
20
+ Use the same code scope and severity model for both analyses. Deduplicate findings by root cause,
21
+ verify every location against the current checkout, and retain a single-model finding when source
22
+ evidence confirms it. Never derive severity from model agreement alone.
56
23
 
57
24
  ## 4. Estimate (`estimate`)
58
25
 
59
- For effort estimation, complexity assessment, or timeline planning.
60
-
61
- **Flow:**
62
- 1. Describe the task, requirements, and constraints to all three AIs
63
- 2. Each AI provides: estimate (time/effort), breakdown, assumptions, risks
64
- 3. Synthesis evaluates convergence:
65
- - All within 20% of each other → **High confidence** estimate (use average)
66
- - One outlier → **Moderate confidence** — investigate the outlier's reasoning
67
- - All significantly different → **Low confidence** — task needs decomposition
68
-
69
- **Output structure:**
70
- - Per-AI estimate with breakdown and assumptions
71
- - Convergence analysis
72
- - Recommended estimate with confidence level
73
- - Risk factors that could shift the estimate
26
+ Require a breakdown, assumptions, dependencies, and risk range from both analyses. Explain large
27
+ differences by locating mismatched scope or assumptions. Return a range supported by the reconciled
28
+ work breakdown rather than averaging estimates mechanically.