prism-mcp-server 7.3.1 → 7.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -19
- package/dist/cli.js +50 -0
- package/dist/darkfactory/runner.js +101 -2
- package/dist/dashboard/ui.js +2617 -2051
- package/dist/dashboard/ui.tmp.js +3475 -0
- package/dist/errors.js +29 -0
- package/dist/storage/sqlite.js +155 -0
- package/dist/storage/supabase.js +116 -0
- package/dist/tools/routerExperience.js +14 -0
- package/dist/verification/clawValidator.js +2 -1
- package/dist/verification/cliHandler.js +325 -0
- package/dist/verification/gatekeeper.js +39 -0
- package/dist/verification/renameDetector.js +170 -0
- package/dist/verification/runner.js +27 -5
- package/dist/verification/schema.js +18 -0
- package/dist/verification/severityPolicy.js +5 -1
- package/package.json +4 -1
package/dist/errors.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thrown when a verification harness gate action evaluates to "abort".
|
|
3
|
+
* Indicates a strict security or operational policy failure that should halt
|
|
4
|
+
* downstream execution immediately.
|
|
5
|
+
*/
|
|
6
|
+
export class VerificationGateError extends Error {
|
|
7
|
+
result;
|
|
8
|
+
constructor(message, result) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "VerificationGateError";
|
|
11
|
+
this.result = result;
|
|
12
|
+
// Maintain V8 stack trace natively
|
|
13
|
+
if (Error.captureStackTrace) {
|
|
14
|
+
Error.captureStackTrace(this, VerificationGateError);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Helper to dump the JSON representation of the failure block for logs.
|
|
19
|
+
*/
|
|
20
|
+
toJSON() {
|
|
21
|
+
return {
|
|
22
|
+
message: this.message,
|
|
23
|
+
project: this.result.project,
|
|
24
|
+
critical_failures: this.result.critical_failures,
|
|
25
|
+
pass_rate: this.result.pass_rate,
|
|
26
|
+
result_json: this.result.result_json,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
}
|
package/dist/storage/sqlite.js
CHANGED
|
@@ -574,6 +574,71 @@ export class SqliteStorage {
|
|
|
574
574
|
)
|
|
575
575
|
`);
|
|
576
576
|
await this.db.execute(`CREATE INDEX IF NOT EXISTS idx_pipelines_status ON dark_factory_pipelines(user_id, project, status)`);
|
|
577
|
+
// ─── v7.2.0 Migration: Verification Harness ────────────────
|
|
578
|
+
await this.db.execute(`
|
|
579
|
+
CREATE TABLE IF NOT EXISTS verification_harnesses (
|
|
580
|
+
rubric_hash TEXT PRIMARY KEY,
|
|
581
|
+
project TEXT NOT NULL,
|
|
582
|
+
conversation_id TEXT NOT NULL,
|
|
583
|
+
created_at TEXT NOT NULL,
|
|
584
|
+
min_pass_rate REAL NOT NULL,
|
|
585
|
+
tests TEXT NOT NULL,
|
|
586
|
+
metadata TEXT,
|
|
587
|
+
user_id TEXT NOT NULL DEFAULT 'default'
|
|
588
|
+
)
|
|
589
|
+
`);
|
|
590
|
+
await this.db.execute(`
|
|
591
|
+
CREATE TABLE IF NOT EXISTS verification_runs (
|
|
592
|
+
id TEXT PRIMARY KEY,
|
|
593
|
+
rubric_hash TEXT NOT NULL,
|
|
594
|
+
project TEXT NOT NULL,
|
|
595
|
+
conversation_id TEXT NOT NULL,
|
|
596
|
+
run_at TEXT NOT NULL,
|
|
597
|
+
passed INTEGER NOT NULL,
|
|
598
|
+
pass_rate REAL NOT NULL,
|
|
599
|
+
critical_failures INTEGER NOT NULL,
|
|
600
|
+
coverage_score REAL NOT NULL,
|
|
601
|
+
result_json TEXT NOT NULL,
|
|
602
|
+
gate_action TEXT NOT NULL,
|
|
603
|
+
gate_override INTEGER,
|
|
604
|
+
override_reason TEXT,
|
|
605
|
+
user_id TEXT NOT NULL DEFAULT 'default',
|
|
606
|
+
FOREIGN KEY(rubric_hash) REFERENCES verification_harnesses(rubric_hash)
|
|
607
|
+
)
|
|
608
|
+
`);
|
|
609
|
+
await this.db.execute(`CREATE INDEX IF NOT EXISTS idx_verification_runs_project ON verification_runs(project, run_at DESC)`);
|
|
610
|
+
// ─── v7.3 Migration: Pipeline Orchestration Overrides ────────
|
|
611
|
+
try {
|
|
612
|
+
await this.db.execute(`ALTER TABLE verification_runs ADD COLUMN gate_override INTEGER`);
|
|
613
|
+
}
|
|
614
|
+
catch (e) {
|
|
615
|
+
if (!e.message?.includes('duplicate column name'))
|
|
616
|
+
console.warn('Migration warning:', e.message);
|
|
617
|
+
}
|
|
618
|
+
try {
|
|
619
|
+
await this.db.execute(`ALTER TABLE verification_runs ADD COLUMN override_reason TEXT`);
|
|
620
|
+
}
|
|
621
|
+
catch (e) {
|
|
622
|
+
if (!e.message?.includes('duplicate column name'))
|
|
623
|
+
console.warn('Migration warning:', e.message);
|
|
624
|
+
}
|
|
625
|
+
// ─── H7 Migration: Tenant isolation for verification tables ────────
|
|
626
|
+
try {
|
|
627
|
+
await this.db.execute(`ALTER TABLE verification_harnesses ADD COLUMN user_id TEXT NOT NULL DEFAULT 'default'`);
|
|
628
|
+
}
|
|
629
|
+
catch (e) {
|
|
630
|
+
if (!e.message?.includes('duplicate column name'))
|
|
631
|
+
console.warn('Migration warning:', e.message);
|
|
632
|
+
}
|
|
633
|
+
try {
|
|
634
|
+
await this.db.execute(`ALTER TABLE verification_runs ADD COLUMN user_id TEXT NOT NULL DEFAULT 'default'`);
|
|
635
|
+
}
|
|
636
|
+
catch (e) {
|
|
637
|
+
if (!e.message?.includes('duplicate column name'))
|
|
638
|
+
console.warn('Migration warning:', e.message);
|
|
639
|
+
}
|
|
640
|
+
// H7: Create index after the column exists (post-migration)
|
|
641
|
+
await this.db.execute(`CREATE INDEX IF NOT EXISTS idx_verification_runs_user ON verification_runs(user_id, project)`);
|
|
577
642
|
// ─── v6.1 Migration: Integrity Check ──────────────────────
|
|
578
643
|
//
|
|
579
644
|
// REVIEWER NOTE: PRAGMA integrity_check scans the B-tree structure of
|
|
@@ -2874,4 +2939,94 @@ export class SqliteStorage {
|
|
|
2874
2939
|
const result = await this.db.execute({ sql, args });
|
|
2875
2940
|
return result.rows;
|
|
2876
2941
|
}
|
|
2942
|
+
// ─── Verification Harness (v7.2.0) ───────────────────────────
|
|
2943
|
+
async saveVerificationHarness(harness, userId) {
|
|
2944
|
+
await this.db.execute({
|
|
2945
|
+
sql: `
|
|
2946
|
+
INSERT INTO verification_harnesses (rubric_hash, project, conversation_id, created_at, min_pass_rate, tests, metadata, user_id)
|
|
2947
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
2948
|
+
ON CONFLICT(rubric_hash) DO UPDATE SET
|
|
2949
|
+
metadata = excluded.metadata
|
|
2950
|
+
`,
|
|
2951
|
+
args: [
|
|
2952
|
+
harness.rubric_hash,
|
|
2953
|
+
harness.project,
|
|
2954
|
+
harness.conversation_id,
|
|
2955
|
+
harness.created_at,
|
|
2956
|
+
harness.min_pass_rate,
|
|
2957
|
+
JSON.stringify(harness.tests),
|
|
2958
|
+
harness.metadata ? JSON.stringify(harness.metadata) : null,
|
|
2959
|
+
userId
|
|
2960
|
+
]
|
|
2961
|
+
});
|
|
2962
|
+
}
|
|
2963
|
+
async getVerificationHarness(rubric_hash, userId) {
|
|
2964
|
+
const result = await this.db.execute({
|
|
2965
|
+
sql: `SELECT * FROM verification_harnesses WHERE rubric_hash = ? AND user_id = ?`,
|
|
2966
|
+
args: [rubric_hash, userId]
|
|
2967
|
+
});
|
|
2968
|
+
if (result.rows.length === 0)
|
|
2969
|
+
return null;
|
|
2970
|
+
const row = result.rows[0];
|
|
2971
|
+
return {
|
|
2972
|
+
...row,
|
|
2973
|
+
tests: JSON.parse(row.tests),
|
|
2974
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined
|
|
2975
|
+
};
|
|
2976
|
+
}
|
|
2977
|
+
async saveVerificationRun(result, userId) {
|
|
2978
|
+
await this.db.execute({
|
|
2979
|
+
sql: `
|
|
2980
|
+
INSERT INTO verification_runs (
|
|
2981
|
+
id, rubric_hash, project, conversation_id, run_at,
|
|
2982
|
+
passed, pass_rate, critical_failures, coverage_score, result_json, gate_action, gate_override, override_reason, user_id
|
|
2983
|
+
)
|
|
2984
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2985
|
+
ON CONFLICT(id) DO NOTHING
|
|
2986
|
+
`,
|
|
2987
|
+
args: [
|
|
2988
|
+
result.id,
|
|
2989
|
+
result.rubric_hash,
|
|
2990
|
+
result.project,
|
|
2991
|
+
result.conversation_id,
|
|
2992
|
+
result.run_at,
|
|
2993
|
+
result.passed ? 1 : 0,
|
|
2994
|
+
result.pass_rate,
|
|
2995
|
+
result.critical_failures,
|
|
2996
|
+
result.coverage_score,
|
|
2997
|
+
result.result_json,
|
|
2998
|
+
result.gate_action,
|
|
2999
|
+
result.gate_override ? 1 : 0,
|
|
3000
|
+
result.override_reason || null,
|
|
3001
|
+
userId
|
|
3002
|
+
]
|
|
3003
|
+
});
|
|
3004
|
+
}
|
|
3005
|
+
async listVerificationRuns(project, userId) {
|
|
3006
|
+
const result = await this.db.execute({
|
|
3007
|
+
sql: `SELECT * FROM verification_runs WHERE project = ? AND user_id = ? ORDER BY run_at DESC`,
|
|
3008
|
+
args: [project, userId]
|
|
3009
|
+
});
|
|
3010
|
+
return result.rows.map(row => ({
|
|
3011
|
+
...row,
|
|
3012
|
+
passed: Boolean(row.passed),
|
|
3013
|
+
gate_override: row.gate_override === 1,
|
|
3014
|
+
override_reason: row.override_reason || undefined
|
|
3015
|
+
}));
|
|
3016
|
+
}
|
|
3017
|
+
async getVerificationRun(id, userId) {
|
|
3018
|
+
const result = await this.db.execute({
|
|
3019
|
+
sql: `SELECT * FROM verification_runs WHERE id = ? AND user_id = ?`,
|
|
3020
|
+
args: [id, userId]
|
|
3021
|
+
});
|
|
3022
|
+
if (result.rows.length === 0)
|
|
3023
|
+
return null;
|
|
3024
|
+
const row = result.rows[0];
|
|
3025
|
+
return {
|
|
3026
|
+
...row,
|
|
3027
|
+
passed: Boolean(row.passed),
|
|
3028
|
+
gate_override: row.gate_override === 1,
|
|
3029
|
+
override_reason: row.override_reason || undefined
|
|
3030
|
+
};
|
|
3031
|
+
}
|
|
2877
3032
|
}
|
package/dist/storage/supabase.js
CHANGED
|
@@ -1271,4 +1271,120 @@ export class SupabaseStorage {
|
|
|
1271
1271
|
throw e;
|
|
1272
1272
|
}
|
|
1273
1273
|
}
|
|
1274
|
+
// ─── Verification Harness (v7.2.0) ───────────────────────────
|
|
1275
|
+
async saveVerificationHarness(harness, userId) {
|
|
1276
|
+
try {
|
|
1277
|
+
await supabasePost("verification_harnesses", {
|
|
1278
|
+
rubric_hash: harness.rubric_hash,
|
|
1279
|
+
project: harness.project,
|
|
1280
|
+
conversation_id: harness.conversation_id,
|
|
1281
|
+
created_at: harness.created_at,
|
|
1282
|
+
min_pass_rate: harness.min_pass_rate,
|
|
1283
|
+
tests: JSON.stringify(harness.tests),
|
|
1284
|
+
metadata: harness.metadata ? JSON.stringify(harness.metadata) : null,
|
|
1285
|
+
user_id: userId
|
|
1286
|
+
}, { on_conflict: "rubric_hash" }, { Prefer: "return=representation,resolution=merge-duplicates" });
|
|
1287
|
+
}
|
|
1288
|
+
catch (e) {
|
|
1289
|
+
if (e.message?.includes("PGRST116") || e.message?.includes("duplicate key")) {
|
|
1290
|
+
return;
|
|
1291
|
+
}
|
|
1292
|
+
throw e;
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
async getVerificationHarness(rubric_hash, userId) {
|
|
1296
|
+
try {
|
|
1297
|
+
const rows = await supabaseGet("verification_harnesses", {
|
|
1298
|
+
"rubric_hash": `eq.${rubric_hash}`,
|
|
1299
|
+
"user_id": `eq.${userId}`
|
|
1300
|
+
});
|
|
1301
|
+
if (!Array.isArray(rows) || rows.length === 0)
|
|
1302
|
+
return null;
|
|
1303
|
+
const row = rows[0];
|
|
1304
|
+
return {
|
|
1305
|
+
...row,
|
|
1306
|
+
tests: JSON.parse(row.tests),
|
|
1307
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
catch (e) {
|
|
1311
|
+
if (e.message?.includes("PGRST202") || e.message?.includes("Could not find the relation"))
|
|
1312
|
+
return null;
|
|
1313
|
+
throw e;
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
async saveVerificationRun(result, userId) {
|
|
1317
|
+
try {
|
|
1318
|
+
await supabasePost("verification_runs", {
|
|
1319
|
+
id: result.id,
|
|
1320
|
+
rubric_hash: result.rubric_hash,
|
|
1321
|
+
project: result.project,
|
|
1322
|
+
conversation_id: result.conversation_id,
|
|
1323
|
+
run_at: result.run_at,
|
|
1324
|
+
// H2 fix: Use native booleans for Supabase/PostgreSQL (not 0/1 integers)
|
|
1325
|
+
passed: result.passed,
|
|
1326
|
+
pass_rate: result.pass_rate,
|
|
1327
|
+
critical_failures: result.critical_failures,
|
|
1328
|
+
coverage_score: result.coverage_score,
|
|
1329
|
+
result_json: result.result_json,
|
|
1330
|
+
gate_action: result.gate_action,
|
|
1331
|
+
gate_override: result.gate_override ?? false,
|
|
1332
|
+
override_reason: result.override_reason || null,
|
|
1333
|
+
user_id: userId
|
|
1334
|
+
}, { on_conflict: "id" }, { Prefer: "return=representation,resolution=ignore-duplicates" });
|
|
1335
|
+
}
|
|
1336
|
+
catch (e) {
|
|
1337
|
+
if (e.message?.includes("PGRST116") || e.message?.includes("duplicate key")) {
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
throw e;
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
async listVerificationRuns(project, userId) {
|
|
1344
|
+
try {
|
|
1345
|
+
const query = {
|
|
1346
|
+
project: `eq.${project}`,
|
|
1347
|
+
user_id: `eq.${userId}`,
|
|
1348
|
+
order: "run_at.desc"
|
|
1349
|
+
};
|
|
1350
|
+
const rows = await supabaseGet("verification_runs", query);
|
|
1351
|
+
if (!Array.isArray(rows))
|
|
1352
|
+
return [];
|
|
1353
|
+
return rows.map((row) => ({
|
|
1354
|
+
...row,
|
|
1355
|
+
passed: Boolean(row.passed),
|
|
1356
|
+
// H2 fix: Use Boolean() consistently (native booleans from Supabase)
|
|
1357
|
+
gate_override: Boolean(row.gate_override),
|
|
1358
|
+
override_reason: row.override_reason || undefined
|
|
1359
|
+
}));
|
|
1360
|
+
}
|
|
1361
|
+
catch (e) {
|
|
1362
|
+
if (e.message?.includes("PGRST202") || e.message?.includes("Could not find the relation"))
|
|
1363
|
+
return [];
|
|
1364
|
+
throw e;
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
async getVerificationRun(id, userId) {
|
|
1368
|
+
try {
|
|
1369
|
+
const rows = await supabaseGet("verification_runs", {
|
|
1370
|
+
id: `eq.${id}`,
|
|
1371
|
+
user_id: `eq.${userId}`
|
|
1372
|
+
});
|
|
1373
|
+
if (!Array.isArray(rows) || rows.length === 0)
|
|
1374
|
+
return null;
|
|
1375
|
+
const row = rows[0];
|
|
1376
|
+
return {
|
|
1377
|
+
...row,
|
|
1378
|
+
passed: Boolean(row.passed),
|
|
1379
|
+
// H2 fix: Use Boolean() consistently (native booleans from Supabase)
|
|
1380
|
+
gate_override: Boolean(row.gate_override),
|
|
1381
|
+
override_reason: row.override_reason || undefined
|
|
1382
|
+
};
|
|
1383
|
+
}
|
|
1384
|
+
catch (e) {
|
|
1385
|
+
if (e.message?.includes("PGRST202") || e.message?.includes("Could not find the relation"))
|
|
1386
|
+
return null;
|
|
1387
|
+
throw e;
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1274
1390
|
}
|
|
@@ -54,6 +54,20 @@ export async function getExperienceBias(project, taskKeywords, storageBackend) {
|
|
|
54
54
|
relevantCount++;
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
|
+
// GAP-1 fix: Ingest validation_result events into ML routing bias.
|
|
58
|
+
// The v7.2 spec requires that "Router learning ingests raw verification
|
|
59
|
+
// signals (pass_rate, critical_failures, coverage_score, rubric_hash)."
|
|
60
|
+
// confidence_score >= 80 indicates a passing verification suite.
|
|
61
|
+
if (eventType === "validation_result") {
|
|
62
|
+
const confidence = raw.confidence_score || 50;
|
|
63
|
+
if (confidence >= 80) {
|
|
64
|
+
successCount++;
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
failureCount++;
|
|
68
|
+
}
|
|
69
|
+
relevantCount++;
|
|
70
|
+
}
|
|
57
71
|
}
|
|
58
72
|
if (relevantCount < MIN_SAMPLES) {
|
|
59
73
|
return {
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
* - claw-code-agent MCP server must be available
|
|
18
18
|
* - PRISM_VERIFICATION_HARNESS_ENABLED=true
|
|
19
19
|
*/
|
|
20
|
+
import { createHash } from "crypto";
|
|
20
21
|
import { TestSuiteSchema } from "./schema.js";
|
|
21
22
|
/**
|
|
22
23
|
* Build the prompt for Claw validation.
|
|
@@ -213,7 +214,7 @@ export function mergeSuggestedAssertions(suite, suggestions) {
|
|
|
213
214
|
...suite.tests,
|
|
214
215
|
...suggestions.map((s) => ({
|
|
215
216
|
...s,
|
|
216
|
-
id: s.id || `claw-suggestion-${
|
|
217
|
+
id: s.id || `claw-suggestion-${createHash("sha256").update(JSON.stringify(s)).digest("hex").slice(0, 12)}`,
|
|
217
218
|
severity: s.severity || "warn",
|
|
218
219
|
})),
|
|
219
220
|
],
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
import * as fs from 'fs/promises';
|
|
2
|
+
import { computeRubricHash } from './schema.js';
|
|
3
|
+
// ─── Constants ────────────────────────────────────────────────────────────────
|
|
4
|
+
/** H5 fix: Centralize the harness file path as a constant */
|
|
5
|
+
const DEFAULT_HARNESS_PATH = './verification_harness.json';
|
|
6
|
+
// ─── Utilities ────────────────────────────────────────────────────────────────
|
|
7
|
+
/** M11 fix: Extract CI environment detection into a reusable utility */
|
|
8
|
+
export function isStrictVerificationEnv() {
|
|
9
|
+
return (process.env.CI === 'true' ||
|
|
10
|
+
process.env.CI === '1' ||
|
|
11
|
+
process.env.GITHUB_ACTIONS === 'true' ||
|
|
12
|
+
process.env.GITLAB_CI === 'true' ||
|
|
13
|
+
process.env.PRISM_STRICT_VERIFICATION === 'true');
|
|
14
|
+
}
|
|
15
|
+
// ─── Renderers ────────────────────────────────────────────────────────────────
|
|
16
|
+
/** Render a VerifyStatusResult as human-readable console output */
|
|
17
|
+
function renderVerifyStatus(result, jsonMode) {
|
|
18
|
+
if (jsonMode) {
|
|
19
|
+
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
console.log(`\n🔍 Checking verification status for project: ${result.project}...`);
|
|
23
|
+
if (result.no_runs) {
|
|
24
|
+
console.log('⚠️ No previous verification runs found.');
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const r = result.last_run;
|
|
28
|
+
const overrideBadge = r.gate_override
|
|
29
|
+
? `[OVERRIDDEN${r.override_reason ? `: ${r.override_reason}` : ''}] `
|
|
30
|
+
: '';
|
|
31
|
+
const passText = r.passed ? 'YES' : 'NO';
|
|
32
|
+
console.log(`✅ Last Run: ${r.run_at} | Passed: ${overrideBadge}${passText}`);
|
|
33
|
+
console.log(` Pass Rate: ${(r.pass_rate * 100).toFixed(1)}% | Critical Failures: ${r.critical_failures}`);
|
|
34
|
+
console.log(` Coverage Score: ${(r.coverage_score * 100).toFixed(1)}% | Gate Action: ${r.gate_action}`);
|
|
35
|
+
if (result.harness_missing) {
|
|
36
|
+
console.log('\nℹ️ No local verification_harness.json found to check against.');
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
if (result.harness_invalid_json) {
|
|
40
|
+
console.error(`\n❌ Invalid JSON in ${DEFAULT_HARNESS_PATH}.`);
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
if (result.synchronized) {
|
|
44
|
+
console.log('\n✨ Harness is synchronized.');
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
// Drift output — phrasing differs only by policy outcome, not unrelated wording
|
|
48
|
+
const d = result.drift;
|
|
49
|
+
const hashLine = ` Stored Hash: ${d.stored_hash.slice(0, 8)}... Local Hash: ${d.local_hash.slice(0, 8)}...`;
|
|
50
|
+
if (d.policy === 'bypassed') {
|
|
51
|
+
console.warn('\n🚨 [BYPASSED] Configuration drift detected.');
|
|
52
|
+
console.warn(hashLine);
|
|
53
|
+
console.warn(` Drift block bypassed via --force. Recommended: run 'prism verify generate' to realign.`);
|
|
54
|
+
}
|
|
55
|
+
else if (d.policy === 'blocked') {
|
|
56
|
+
console.error('\n🚫 [BLOCKED] Configuration drift detected — CI environment enforces strict policy.');
|
|
57
|
+
console.error(hashLine);
|
|
58
|
+
console.error(` Action: run 'prism verify generate' before merging to update your harness.`);
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
// 'warn' — local dev
|
|
62
|
+
console.warn('\n⚠️ [DRIFT] Configuration drift detected.');
|
|
63
|
+
console.warn(hashLine);
|
|
64
|
+
console.warn(` Recommended: run 'prism verify generate' to update your harness.`);
|
|
65
|
+
}
|
|
66
|
+
// Render Diff if available
|
|
67
|
+
if (d.diff_counts) {
|
|
68
|
+
console.log(`\n Diff Summary: +${d.diff_counts.added} added, ~${d.diff_counts.modified} modified, -${d.diff_counts.removed} removed`);
|
|
69
|
+
}
|
|
70
|
+
if (d.diff) {
|
|
71
|
+
console.log('\n Changes Detected:');
|
|
72
|
+
for (const add of d.diff.added)
|
|
73
|
+
console.log(` + ${add.id}: ${add.description}`);
|
|
74
|
+
for (const mod of d.diff.modified) {
|
|
75
|
+
const keys = mod.changed_keys;
|
|
76
|
+
const keySuffix = keys?.length ? ` [${keys.join(', ')}]` : '';
|
|
77
|
+
console.log(` ~ ${mod.id}: ${mod.description}${keySuffix}`);
|
|
78
|
+
}
|
|
79
|
+
for (const rem of d.diff.removed)
|
|
80
|
+
console.log(` - ${rem.id}: ${rem.description}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Render a GenerateHarnessResult as human-readable console output */
|
|
84
|
+
function renderGenerateHarness(result, jsonMode) {
|
|
85
|
+
if (jsonMode) {
|
|
86
|
+
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
console.log(`\n🛠 Generating/Refreshing harness for project: ${result.project}...`);
|
|
90
|
+
if (result.file_missing) {
|
|
91
|
+
console.error(`❌ Failed to read ${DEFAULT_HARNESS_PATH}. Does the file exist?`);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
if (result.invalid_json) {
|
|
95
|
+
console.error(`❌ Invalid JSON in ${DEFAULT_HARNESS_PATH}.`);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
if (result.already_exists) {
|
|
99
|
+
console.warn(`\n⚠️ A harness with rubric hash ${result.rubric_hash?.slice(0, 12)}... already exists.`);
|
|
100
|
+
console.warn(' Use --force to re-register anyway.');
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (result.success) {
|
|
104
|
+
console.log('✅ Harness registered successfully.');
|
|
105
|
+
console.log(` Hash: ${result.rubric_hash?.slice(0, 12)}...`);
|
|
106
|
+
console.log(` Tests: ${result.test_count} assertions.`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// ─── Handlers ─────────────────────────────────────────────────────────────────
|
|
110
|
+
/**
|
|
111
|
+
* Core logic for `verify status`.
|
|
112
|
+
* Returns a typed VerifyStatusResult — callers decide how to render/exit.
|
|
113
|
+
*/
|
|
114
|
+
export async function computeVerifyStatus(storage, project, force = false, userId = 'default') {
|
|
115
|
+
const base = {
|
|
116
|
+
schema_version: 1,
|
|
117
|
+
project,
|
|
118
|
+
no_runs: false,
|
|
119
|
+
harness_missing: false,
|
|
120
|
+
harness_invalid_json: false,
|
|
121
|
+
synchronized: null,
|
|
122
|
+
recommended_action: null,
|
|
123
|
+
exit_code: 0,
|
|
124
|
+
};
|
|
125
|
+
// 1. Get latest run
|
|
126
|
+
const runs = await storage.listVerificationRuns(project, userId);
|
|
127
|
+
const lastRun = runs[0];
|
|
128
|
+
if (!lastRun) {
|
|
129
|
+
return { ...base, no_runs: true, recommended_action: 'run prism verify generate' };
|
|
130
|
+
}
|
|
131
|
+
base.last_run = {
|
|
132
|
+
run_at: lastRun.run_at,
|
|
133
|
+
passed: lastRun.passed,
|
|
134
|
+
pass_rate: lastRun.pass_rate,
|
|
135
|
+
critical_failures: lastRun.critical_failures,
|
|
136
|
+
coverage_score: lastRun.coverage_score,
|
|
137
|
+
gate_action: lastRun.gate_action,
|
|
138
|
+
gate_override: lastRun.gate_override ?? false,
|
|
139
|
+
override_reason: lastRun.override_reason,
|
|
140
|
+
};
|
|
141
|
+
// 2. Drift detection — C5 fix: separate readFile and JSON.parse error paths
|
|
142
|
+
let harnessRaw;
|
|
143
|
+
try {
|
|
144
|
+
harnessRaw = await fs.readFile(DEFAULT_HARNESS_PATH, 'utf-8');
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return { ...base, harness_missing: true };
|
|
148
|
+
}
|
|
149
|
+
let localHarness;
|
|
150
|
+
try {
|
|
151
|
+
localHarness = JSON.parse(harnessRaw);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return { ...base, harness_invalid_json: true, exit_code: 1 };
|
|
155
|
+
}
|
|
156
|
+
const localHash = computeRubricHash(localHarness.tests);
|
|
157
|
+
const storedHash = lastRun.rubric_hash;
|
|
158
|
+
if (localHash === storedHash) {
|
|
159
|
+
return { ...base, synchronized: true };
|
|
160
|
+
}
|
|
161
|
+
// Drift detected
|
|
162
|
+
const strictEnv = isStrictVerificationEnv();
|
|
163
|
+
// Phase 2 Diagnostics: Compute Structured Diff
|
|
164
|
+
let diff;
|
|
165
|
+
let diffCounts;
|
|
166
|
+
try {
|
|
167
|
+
const historicalHarness = await storage.getVerificationHarness?.(storedHash, userId);
|
|
168
|
+
if (historicalHarness) {
|
|
169
|
+
diff = { added: [], removed: [], modified: [] };
|
|
170
|
+
const dbTests = historicalHarness.tests;
|
|
171
|
+
const localTests = localHarness.tests;
|
|
172
|
+
const storedMap = new Map(dbTests.map(t => [t.id, t]));
|
|
173
|
+
const localMap = new Map(localTests.map(t => [t.id, t]));
|
|
174
|
+
for (const [id, localTest] of localMap.entries()) {
|
|
175
|
+
const storedTest = storedMap.get(id);
|
|
176
|
+
if (!storedTest) {
|
|
177
|
+
diff.added.push(localTest);
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
// Compare JSON stringification for deep equality
|
|
181
|
+
const storedStr = JSON.stringify(storedTest);
|
|
182
|
+
const localStr = JSON.stringify(localTest);
|
|
183
|
+
if (storedStr !== localStr) {
|
|
184
|
+
// Diagnostics v2: Compute changed_keys — top-level fields that differ
|
|
185
|
+
const allKeys = new Set([...Object.keys(storedTest), ...Object.keys(localTest)]);
|
|
186
|
+
const changedKeys = [];
|
|
187
|
+
for (const key of allKeys) {
|
|
188
|
+
if (JSON.stringify(storedTest[key]) !== JSON.stringify(localTest[key])) {
|
|
189
|
+
changedKeys.push(key);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
changedKeys.sort();
|
|
193
|
+
diff.modified.push({ ...localTest, changed_keys: changedKeys });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
for (const [id, storedTest] of storedMap.entries()) {
|
|
198
|
+
if (!localMap.has(id)) {
|
|
199
|
+
diff.removed.push(storedTest);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
// Ensure stable ordering by ID
|
|
203
|
+
diff.added.sort((a, b) => a.id.localeCompare(b.id));
|
|
204
|
+
diff.removed.sort((a, b) => a.id.localeCompare(b.id));
|
|
205
|
+
diff.modified.sort((a, b) => a.id.localeCompare(b.id));
|
|
206
|
+
// Diagnostics v2: Compute summary counts
|
|
207
|
+
diffCounts = {
|
|
208
|
+
added: diff.added.length,
|
|
209
|
+
removed: diff.removed.length,
|
|
210
|
+
modified: diff.modified.length,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
// Failure to load historical harness skips structured diff output (safe default)
|
|
216
|
+
// Downstream consumers parse JSON and know `diff`/`diff_counts` are optional per schema constraints.
|
|
217
|
+
}
|
|
218
|
+
const driftBase = (diff && diffCounts)
|
|
219
|
+
? { stored_hash: storedHash, local_hash: localHash, strict_env: strictEnv, diff, diff_counts: diffCounts }
|
|
220
|
+
: { stored_hash: storedHash, local_hash: localHash, strict_env: strictEnv };
|
|
221
|
+
const action = "run 'prism verify generate' to update your harness";
|
|
222
|
+
if (force) {
|
|
223
|
+
return {
|
|
224
|
+
...base,
|
|
225
|
+
synchronized: false,
|
|
226
|
+
drift: { ...driftBase, policy: 'bypassed' },
|
|
227
|
+
recommended_action: action,
|
|
228
|
+
exit_code: 0,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
if (strictEnv) {
|
|
232
|
+
return {
|
|
233
|
+
...base,
|
|
234
|
+
synchronized: false,
|
|
235
|
+
drift: { ...driftBase, policy: 'blocked' },
|
|
236
|
+
recommended_action: action,
|
|
237
|
+
exit_code: 1,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
return {
|
|
241
|
+
...base,
|
|
242
|
+
synchronized: false,
|
|
243
|
+
drift: { ...driftBase, policy: 'warn' },
|
|
244
|
+
recommended_action: action,
|
|
245
|
+
exit_code: 0,
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* CLI entry-point for `verify status`.
|
|
250
|
+
* Computes the result, renders it (human or JSON), then sets process.exitCode.
|
|
251
|
+
*/
|
|
252
|
+
export async function handleVerifyStatus(storage, project, force = false, userId = 'default', jsonMode = false) {
|
|
253
|
+
const result = await computeVerifyStatus(storage, project, force, userId);
|
|
254
|
+
renderVerifyStatus(result, jsonMode);
|
|
255
|
+
// Use process.exitCode rather than process.exit() for cleaner test teardown
|
|
256
|
+
if (result.exit_code !== 0) {
|
|
257
|
+
process.exitCode = result.exit_code;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Core logic for `verify generate`.
|
|
262
|
+
* Returns a typed GenerateHarnessResult.
|
|
263
|
+
*/
|
|
264
|
+
export async function computeGenerateHarness(storage, project, force = false, userId = 'default') {
|
|
265
|
+
const base = {
|
|
266
|
+
schema_version: 1,
|
|
267
|
+
project,
|
|
268
|
+
success: false,
|
|
269
|
+
already_exists: false,
|
|
270
|
+
file_missing: false,
|
|
271
|
+
invalid_json: false,
|
|
272
|
+
exit_code: 0,
|
|
273
|
+
};
|
|
274
|
+
let raw;
|
|
275
|
+
try {
|
|
276
|
+
raw = await fs.readFile(DEFAULT_HARNESS_PATH, 'utf-8');
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
return { ...base, file_missing: true, exit_code: 1 };
|
|
280
|
+
}
|
|
281
|
+
let harnessData;
|
|
282
|
+
try {
|
|
283
|
+
harnessData = JSON.parse(raw);
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
return { ...base, invalid_json: true, exit_code: 1 };
|
|
287
|
+
}
|
|
288
|
+
const rubric_hash = computeRubricHash(harnessData.tests);
|
|
289
|
+
// H3 fix: If not --force, check if a harness already exists for this hash
|
|
290
|
+
if (!force) {
|
|
291
|
+
try {
|
|
292
|
+
const existing = await storage.getVerificationHarness?.(rubric_hash, userId);
|
|
293
|
+
if (existing) {
|
|
294
|
+
return { ...base, already_exists: true, rubric_hash, exit_code: 0 };
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
// getVerificationHarness may not exist on all backends; proceed
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const harness = {
|
|
302
|
+
...harnessData,
|
|
303
|
+
project,
|
|
304
|
+
created_at: new Date().toISOString(),
|
|
305
|
+
rubric_hash,
|
|
306
|
+
};
|
|
307
|
+
await storage.saveVerificationHarness(harness, userId);
|
|
308
|
+
return {
|
|
309
|
+
...base,
|
|
310
|
+
success: true,
|
|
311
|
+
rubric_hash,
|
|
312
|
+
test_count: harness.tests.length,
|
|
313
|
+
exit_code: 0,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* CLI entry-point for `verify generate`.
|
|
318
|
+
*/
|
|
319
|
+
export async function handleGenerateHarness(storage, project, force = false, userId = 'default', jsonMode = false) {
|
|
320
|
+
const result = await computeGenerateHarness(storage, project, force, userId);
|
|
321
|
+
renderGenerateHarness(result, jsonMode);
|
|
322
|
+
if (result.exit_code !== 0) {
|
|
323
|
+
process.exitCode = result.exit_code;
|
|
324
|
+
}
|
|
325
|
+
}
|