qualia-framework 7.2.1 → 7.3.0

package/tests/verify-panel.test.sh

@@ -157,6 +157,242 @@ rm -rf "$TMP"
 $NODE "$VP" /nonexistent/panel.json >/dev/null 2>&1
 assert_exit "missing panel file → exit 2" 2 $?
 
+# --- skeptic tally: pure function is deterministic + ignores stray prose ---
+TALLY1=$($NODE -e "console.log(JSON.stringify(require('$VP').tallySkepticVotes('REAL — reachable\nNOT_REAL — handled\nREAL — yep\n\nrandom prose\nnot_real — lowercase ok')))")
+TALLY2=$($NODE -e "console.log(JSON.stringify(require('$VP').tallySkepticVotes('REAL — reachable\nNOT_REAL — handled\nREAL — yep\n\nrandom prose\nnot_real — lowercase ok')))")
+assert_eq "tally is deterministic (same input → same counts)" "$TALLY1" "$TALLY2"
+assert_contains "tally counts REAL votes" "$TALLY1" '"real":2'
+assert_contains "tally counts NOT_REAL (incl lowercase, not mistaken for REAL)" "$TALLY1" '"notReal":2'
+
+# --- skeptic subcommand: writes votes onto the matching finding (mechanical, no hand-edit) ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/.planning/phase-9-panel.json" <<'EOF'
+{ "phase": 9, "lenses": ["security"], "findings": [
+  { "lens":"security", "file":"lib/auth.ts", "line":42, "severity":"CRITICAL", "title":"service_role reachable client-side", "votes": {"real":0,"notReal":0} }
+] }
+EOF
+KEY="lib/auth.ts:42:service role reachable client side"
+printf '%s\n' "REAL — confirmed reachable" "REAL — still reachable" "NOT_REAL — handled" \
+  | (cd "$TMP" && $NODE "$VP" skeptic 9 "$KEY" >/dev/null 2>&1)
+assert_exit "skeptic subcommand exits 0 on a matching key" 0 $?
+WROTE=$(cat "$TMP/.planning/phase-9-panel.json")
+assert_contains "skeptic wrote real count" "$WROTE" '"real": 2'
+assert_contains "skeptic wrote notReal count" "$WROTE" '"notReal": 1'
+# determinism end-to-end: identical replies → identical counts on re-run
+printf '%s\n' "REAL — confirmed reachable" "REAL — still reachable" "NOT_REAL — handled" \
+  | (cd "$TMP" && $NODE "$VP" skeptic 9 "$KEY" >/dev/null 2>&1)
+WROTE2=$(cat "$TMP/.planning/phase-9-panel.json")
+assert_eq "skeptic tally is reproducible (same replies → same JSON)" "$WROTE" "$WROTE2"
+# the tallied 2✓/1✗ CRITICAL survives (majority-survives) → aggregator still FAILs
+(cd "$TMP" && $NODE "$VP" .planning/phase-9-panel.json >/dev/null 2>&1)
+assert_exit "tallied CRITICAL (2✓/1✗) survives → FAIL" 1 $?
+rm -rf "$TMP"
+
+# --- skeptic subcommand: majority not-real, written by the tally, flips to PASS ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/.planning/phase-9-panel.json" <<'EOF'
+{ "phase": 9, "lenses": ["security"], "findings": [
+  { "lens":"security", "file":"lib/auth.ts", "line":42, "severity":"CRITICAL", "title":"false alarm", "votes": {"real":0,"notReal":0} }
+] }
+EOF
+printf '%s\n' "NOT_REAL — false positive" "NOT_REAL — already guarded" "REAL — unsure" \
+  | (cd "$TMP" && $NODE "$VP" skeptic 9 "lib/auth.ts:42:false alarm" >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" .planning/phase-9-panel.json >/dev/null 2>&1)
+assert_exit "tallied majority NOT_REAL kills CRITICAL → PASS" 0 $?
+rm -rf "$TMP"
+
+# --- skeptic subcommand: unknown key → exit 2 (cannot silently no-op a tally) ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/.planning/phase-9-panel.json" <<'EOF'
+{ "phase": 9, "lenses": ["security"], "findings": [
+  { "lens":"security", "file":"lib/auth.ts", "line":42, "severity":"CRITICAL", "title":"real one", "votes": {"real":0,"notReal":0} }
+] }
+EOF
+printf '%s\n' "REAL — x" | (cd "$TMP" && $NODE "$VP" skeptic 9 "does/not:1:exist" >/dev/null 2>&1)
+assert_exit "skeptic unknown key → exit 2" 2 $?
+rm -rf "$TMP"
+
+# --- execution lens: a failing build script → CRITICAL panel-execution.json that assemble() counts ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/package.json" <<'EOF'
+{ "name": "fixture-red", "scripts": { "build": "exit 7" } }
+EOF
+(cd "$TMP" && $NODE "$VP" execution 11 >/dev/null 2>&1)
+assert_exit "execution lens with a red build → exit 1" 1 $?
+[ -f "$TMP/.planning/phase-11-panel-execution.json" ] && { echo "  ✓ execution lens writes panel-execution.json"; PASS=$((PASS+1)); } || { echo "  ✗ no panel-execution.json"; FAIL=$((FAIL+1)); }
+EXEC=$(cat "$TMP/.planning/phase-11-panel-execution.json")
+assert_contains "execution failure is CRITICAL" "$EXEC" '"severity": "CRITICAL"'
+assert_contains "execution failure names the build" "$EXEC" 'npm run build failed'
+# assemble() folds the execution lens into the panel like any other lens file
+(cd "$TMP" && $NODE "$VP" assemble 11 >/dev/null 2>&1)
+ASM=$(cat "$TMP/.planning/phase-11-panel.json")
+assert_contains "assemble tags execution lens" "$ASM" '"lens": "execution"'
+# and the assembled panel aggregates to FAIL (the CRITICAL survives, unvoted)
+(cd "$TMP" && $NODE "$VP" .planning/phase-11-panel.json >/dev/null 2>&1)
+assert_exit "execution CRITICAL → assembled panel FAILs" 1 $?
+rm -rf "$TMP"
+
+# --- execution lens: absent tools → SKIPPED, not FAIL (fail-soft) ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+# no tsconfig.json, no package.json → every check is inapplicable
+(cd "$TMP" && $NODE "$VP" execution 12 >/dev/null 2>&1)
+assert_exit "execution lens with no checks → exit 0 (fail-soft)" 0 $?
+EXEC=$(cat "$TMP/.planning/phase-12-panel-execution.json")
+assert_eq "absent tools produce an empty finding array (SKIPPED, not FAIL)" "[]" "$(echo "$EXEC" | tr -d '[:space:]')"
+rm -rf "$TMP"
+
+# --- execution lens: a no-op default test script is SKIPPED, not run as a failure ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/package.json" <<'EOF'
+{ "name": "fixture-notest", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" } }
+EOF
+(cd "$TMP" && $NODE "$VP" execution 13 >/dev/null 2>&1)
+assert_exit "npm-init default 'no test specified' is SKIPPED → exit 0" 0 $?
+EXEC=$(cat "$TMP/.planning/phase-13-panel-execution.json")
+assert_eq "default no-op test produces no finding" "[]" "$(echo "$EXEC" | tr -d '[:space:]')"
+rm -rf "$TMP"
+
+# --- execution lens: a passing build script → no finding (present + exit 0 = PASS) ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/package.json" <<'EOF'
+{ "name": "fixture-green", "scripts": { "build": "exit 0" } }
+EOF
+(cd "$TMP" && $NODE "$VP" execution 14 >/dev/null 2>&1)
+assert_exit "execution lens with a green build → exit 0" 0 $?
+EXEC=$(cat "$TMP/.planning/phase-14-panel-execution.json")
+assert_eq "a passing check produces no finding" "[]" "$(echo "$EXEC" | tr -d '[:space:]')"
+rm -rf "$TMP"
+
+# ============================================================================
+# verdict aggregator (ADR-0002) — fold machine-JSON gates into one deterministic
+# PASS/FAIL. The aggregator only makes the existing AND deterministic; it must
+# not start blocking anything that passes today (no-regression principle).
+# ============================================================================
+
+# --- gate recorder: a failing gate's REAL exit code → blocking CRITICAL artifact ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 20 dep-verify --exit 1 >/dev/null 2>&1)
+assert_exit "gate recorder exits 0 (it records, never blocks)" 0 $?
+[ -f "$TMP/.planning/phase-20-gate-dep-verify.json" ] && { echo "  ✓ gate writes phase-20-gate-dep-verify.json"; PASS=$((PASS+1)); } || { echo "  ✗ no gate artifact"; FAIL=$((FAIL+1)); }
+G=$(cat "$TMP/.planning/phase-20-gate-dep-verify.json")
+assert_contains "gate records CRITICAL on non-zero exit" "$G" '"severity": "CRITICAL"'
+assert_contains "gate records blocking:true for dep-verify" "$G" '"blocking": true'
+assert_contains "gate records a finding on failure" "$G" '"findings"'
+rm -rf "$TMP"
+
+# --- gate recorder: exit 0 → clean artifact, empty findings ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 21 slop-detect --exit 0 >/dev/null 2>&1)
+G=$(cat "$TMP/.planning/phase-21-gate-slop-detect.json")
+assert_contains "clean gate has empty findings" "$G" '"findings": []'
+rm -rf "$TMP"
+
+# --- verdict: a blocking gate's block condition flips the verdict to FAIL ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 22 dep-verify --exit 1 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 22 >/dev/null 2>&1)
+assert_exit "blocking gate CRITICAL → verdict FAIL (exit 1)" 1 $?
+OUT=$(cd "$TMP" && $NODE "$VP" verdict 22 --json 2>&1)
+assert_contains "verdict FAIL on blocking gate" "$OUT" '"verdict": "FAIL"'
+rm -rf "$TMP"
+
+# --- verdict: slop-detect block condition (CRITICAL slop, exit 1) also flips ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 23 slop-detect --exit 1 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 23 >/dev/null 2>&1)
+assert_exit "slop-detect CRITICAL → verdict FAIL (exit 1)" 1 $?
+rm -rf "$TMP"
+
+# --- verdict: harness-eval hard FAIL flips; per-suite qualia-eval CRITICAL flips ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 24 harness-eval --exit 1 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 24 >/dev/null 2>&1)
+assert_exit "harness-eval hard FAIL → verdict FAIL" 1 $?
+rm -rf "$TMP"
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 25 eval-rag --exit 1 --title "qualia-eval suite rag FAILED" >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 25 >/dev/null 2>&1)
+assert_exit "per-suite qualia-eval FAIL → verdict FAIL" 1 $?
+OUT=$(cd "$TMP" && $NODE "$VP" verdict 25 --json 2>&1)
+assert_contains "failing suite recorded as its own CRITICAL" "$OUT" 'suite rag FAILED'
+rm -rf "$TMP"
+
+# --- NO REGRESSION: a non-blocking finding (soft harness-eval MEDIUM) does NOT flip ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 26 harness-eval --exit 1 --severity MEDIUM >/dev/null 2>&1)
+G=$(cat "$TMP/.planning/phase-26-gate-harness-eval.json")
+assert_contains "soft sub-check recorded MEDIUM" "$G" '"severity": "MEDIUM"'
+assert_contains "MEDIUM gate is non-blocking" "$G" '"blocking": false'
+(cd "$TMP" && $NODE "$VP" verdict 26 >/dev/null 2>&1)
+assert_exit "non-blocking MEDIUM gate → verdict still PASS (no regression)" 0 $?
+OUT=$(cd "$TMP" && $NODE "$VP" verdict 26 --json 2>&1)
+assert_contains "non-blocking finding still recorded/visible" "$OUT" '"MEDIUM": 1'
+rm -rf "$TMP"
+
+# --- HIGH slop recorded but non-blocking does NOT flip (block condition is CRITICAL) ---
+# slop-detect runs at --severity=critical, so a HIGH-only result exits 0 → clean gate.
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 27 slop-detect --exit 0 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 27 >/dev/null 2>&1)
+assert_exit "HIGH/MEDIUM slop (gate exit 0) → verdict PASS" 0 $?
+rm -rf "$TMP"
+
+# --- verdict: a clean run (all gates exit 0, no panel findings) → PASS ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 28 dep-verify --exit 0 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" gate 28 slop-detect --exit 0 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" gate 28 harness-eval --exit 0 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 28 >/dev/null 2>&1)
+assert_exit "clean run (all gates green) → verdict PASS (exit 0)" 0 $?
+rm -rf "$TMP"
+
+# --- verdict folds the panel lenses AND gates together (execution lens + gate) ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/.planning/phase-29-panel-security.json" <<'EOF'
+[{"file":"auth.ts","line":1,"severity":"CRITICAL","title":"leak"}]
+EOF
+(cd "$TMP" && $NODE "$VP" gate 29 slop-detect --exit 0 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 29 >/dev/null 2>&1)
+assert_exit "panel CRITICAL folds into verdict even with clean gates → FAIL" 1 $?
+OUT=$(cd "$TMP" && $NODE "$VP" verdict 29 --json 2>&1)
+assert_contains "verdict surfaces the panel finding" "$OUT" 'leak'
+rm -rf "$TMP"
+
+# --- DETERMINISM: same artifacts → identical verdict twice ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/.planning/phase-30-panel-correctness.json" <<'EOF'
+[{"file":"a.ts","line":3,"severity":"HIGH","title":"off by one"}]
+EOF
+(cd "$TMP" && $NODE "$VP" gate 30 dep-verify --exit 1 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" gate 30 harness-eval --exit 1 --severity MEDIUM >/dev/null 2>&1)
+V1=$(cd "$TMP" && $NODE "$VP" verdict 30 --json 2>&1)
+V2=$(cd "$TMP" && $NODE "$VP" verdict 30 --json 2>&1)
+assert_eq "verdict is deterministic (same artifacts → identical JSON)" "$V1" "$V2"
+(cd "$TMP" && $NODE "$VP" verdict 30 >/dev/null 2>&1)
+assert_exit "deterministic run with a blocking gate → FAIL both times" 1 $?
+rm -rf "$TMP"
+
+# --- verdict --write emits the verdict artifacts ---
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+(cd "$TMP" && $NODE "$VP" gate 31 dep-verify --exit 1 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 31 --write >/dev/null 2>&1)
+[ -f "$TMP/.planning/phase-31-verdict.json" ] && { echo "  ✓ verdict writes phase-31-verdict.json"; PASS=$((PASS+1)); } || { echo "  ✗ no verdict json"; FAIL=$((FAIL+1)); }
+[ -f "$TMP/.planning/phase-31-verdict.md" ] && { echo "  ✓ verdict writes phase-31-verdict.md"; PASS=$((PASS+1)); } || { echo "  ✗ no verdict md"; FAIL=$((FAIL+1)); }
+rm -rf "$TMP"
+
+# --- verdict honors skeptic votes already on the panel (does not re-run them) ---
+# A panel.json with a CRITICAL skeptics killed (majority not-real) → verdict PASS.
+TMP=$(mktemp -d); mkdir -p "$TMP/.planning"
+cat > "$TMP/.planning/phase-32-panel.json" <<'EOF'
+{ "phase": 32, "lenses": ["security"], "findings": [
+  { "lens":"security", "file":"x.ts", "line":1, "severity":"CRITICAL", "title":"false alarm", "votes": {"real":1,"notReal":2} }
+] }
+EOF
+(cd "$TMP" && $NODE "$VP" gate 32 slop-detect --exit 0 >/dev/null 2>&1)
+(cd "$TMP" && $NODE "$VP" verdict 32 >/dev/null 2>&1)
+assert_exit "verdict honors skeptic-killed CRITICAL → PASS" 0 $?
+rm -rf "$TMP"
+
 echo ""
 echo "=== Results: $PASS passed, $FAIL failed ==="
 [ "$FAIL" -eq 0 ] && exit 0 || exit 1