secufusion-mcp 2.5.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -11
- package/agents/planner.md +136 -119
- package/agents/reviewer.md +133 -0
- package/mcp/dist/benchmark_fixture.js +46 -0
- package/mcp/dist/benchmark_phase4.js +135 -0
- package/mcp/dist/benchmark_phase5.js +121 -0
- package/mcp/dist/benchmark_phase6.js +83 -0
- package/mcp/dist/benchmark_phase7.js +106 -0
- package/mcp/dist/benchmark_phase8.js +75 -0
- package/mcp/dist/benchmark_phase9.js +69 -0
- package/mcp/dist/benchmark_runner.js +130 -0
- package/mcp/dist/benchmark_traceability.js +74 -0
- package/mcp/dist/experience_manager.js +226 -0
- package/mcp/dist/graph_builder.js +148 -0
- package/mcp/dist/implementation_plan.js +92 -0
- package/mcp/dist/investigation_engine.js +59 -0
- package/mcp/dist/regression_engine.js +196 -0
- package/mcp/dist/requirement_manager.js +84 -0
- package/mcp/dist/run_benchmarks.js +62 -0
- package/mcp/dist/server.js +745 -153
- package/mcp/dist/test_engine.js +113 -0
- package/mcp/dist/test_graph.js +60 -0
- package/mcp/dist/traceability_engine.js +106 -0
- package/mcp/dist/trigger_extraction.js +4 -0
- package/mcp/dist/ui_intelligence.js +74 -0
- package/mcp/dist/validation_engine.js +69 -0
- package/mcp/dist/workspace_intelligence.js +99 -0
- package/package.json +1 -1
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { savePlan, validatePlan } from "./implementation_plan.js";
|
|
2
|
+
import { generateTestObligations, aggregateValidation } from "./validation_engine.js";
|
|
3
|
+
import { recordExperience, loadExperience } from "./experience_manager.js";
|
|
4
|
+
import { saveRequirement } from "./requirement_manager.js";
|
|
5
|
+
import { saveTraceability } from "./traceability_engine.js";
|
|
6
|
+
import { performFullExtraction } from "./server.js";
|
|
7
|
+
import fs from 'fs';
|
|
8
|
+
import path from 'path';
|
|
9
|
+
const ecosystemRoot = "c:\\Users\\Yash\\Desktop\\mcp";
|
|
10
|
+
function setupRequirement(id, title, status, decisions = []) {
|
|
11
|
+
const req = {
|
|
12
|
+
id,
|
|
13
|
+
title,
|
|
14
|
+
status,
|
|
15
|
+
raw_request: title,
|
|
16
|
+
clarification_questions: [],
|
|
17
|
+
decisions
|
|
18
|
+
};
|
|
19
|
+
saveRequirement(ecosystemRoot, req);
|
|
20
|
+
}
|
|
21
|
+
function runPhase4Benchmark() {
|
|
22
|
+
console.log("=== RUNNING PHASE 4 VALIDATION BENCHMARK ===");
|
|
23
|
+
// 1. Initial State Setup
|
|
24
|
+
const stats = performFullExtraction(ecosystemRoot);
|
|
25
|
+
const dnaPath = path.join(ecosystemRoot, ".secufusion", "dna.json");
|
|
26
|
+
const dna = JSON.parse(fs.readFileSync(dnaPath, "utf-8"));
|
|
27
|
+
const graph = dna.graph;
|
|
28
|
+
saveTraceability(ecosystemRoot, []);
|
|
29
|
+
// 2. Retention Benchmark (Destructive Operation without Explicit Decision)
|
|
30
|
+
console.log("\\n--- RETENTION: Destructive Operation Guardrails ---");
|
|
31
|
+
setupRequirement("REQ-RET-2", "Implement retention policy cleanup", "CONFIRMED"); // No decisions array has "cleanup"
|
|
32
|
+
let planRet = {
|
|
33
|
+
id: "PLAN-RET",
|
|
34
|
+
requirement_id: "REQ-RET-2",
|
|
35
|
+
status: "DRAFT",
|
|
36
|
+
summary: "Delete old events",
|
|
37
|
+
affected_services: ["sfn-events-api"],
|
|
38
|
+
affected_files: [],
|
|
39
|
+
api_changes: [],
|
|
40
|
+
database_changes: [],
|
|
41
|
+
event_changes: [],
|
|
42
|
+
frontend_changes: [],
|
|
43
|
+
extension_changes: [],
|
|
44
|
+
security_implications: [],
|
|
45
|
+
implementation_steps: ["cleanup events older than 30 days"], // destructive keyword
|
|
46
|
+
test_obligations: ["Test deletion limits"],
|
|
47
|
+
validation_requirements: [],
|
|
48
|
+
traceability_requirements: [],
|
|
49
|
+
risks: [],
|
|
50
|
+
assumptions: [],
|
|
51
|
+
unknowns: [],
|
|
52
|
+
blockers: [],
|
|
53
|
+
created_at: new Date().toISOString(),
|
|
54
|
+
updated_at: new Date().toISOString(),
|
|
55
|
+
snapshot_id: "LATEST"
|
|
56
|
+
};
|
|
57
|
+
savePlan(ecosystemRoot, planRet);
|
|
58
|
+
let errorsRet = validatePlan(ecosystemRoot, planRet);
|
|
59
|
+
console.log("Validation Errors (No Decision):", errorsRet);
|
|
60
|
+
let validationRet = aggregateValidation("CONFIRMED", errorsRet, [], [], [], false);
|
|
61
|
+
console.log("Overall State (No Decision):", validationRet.overall);
|
|
62
|
+
// Now with explicit decision
|
|
63
|
+
setupRequirement("REQ-RET-2", "Implement retention policy cleanup", "CONFIRMED", ["Authorized to cleanup old events"]);
|
|
64
|
+
errorsRet = validatePlan(ecosystemRoot, planRet);
|
|
65
|
+
console.log("Validation Errors (With Decision):", errorsRet);
|
|
66
|
+
validationRet = aggregateValidation("CONFIRMED", errorsRet, [], [], [], false);
|
|
67
|
+
console.log("Overall State (With Decision):", validationRet.overall);
|
|
68
|
+
// 3. User Deletion Benchmark (Test Obligations)
|
|
69
|
+
console.log("\\n--- USER DELETION: Test Intelligence ---");
|
|
70
|
+
setupRequirement("REQ-DEL-2", "User Deletion Feature", "CONFIRMED");
|
|
71
|
+
let planDel = {
|
|
72
|
+
id: "PLAN-DEL",
|
|
73
|
+
requirement_id: "REQ-DEL-2",
|
|
74
|
+
status: "DRAFT",
|
|
75
|
+
summary: "Delete user via API",
|
|
76
|
+
affected_services: ["sfn-iam-api"],
|
|
77
|
+
affected_files: [],
|
|
78
|
+
api_changes: ["DELETE /users/{id}"],
|
|
79
|
+
database_changes: ["DELETE cascade on users table"],
|
|
80
|
+
event_changes: ["Produce user-deleted event"],
|
|
81
|
+
frontend_changes: [],
|
|
82
|
+
extension_changes: [],
|
|
83
|
+
security_implications: ["Requires admin scope"],
|
|
84
|
+
implementation_steps: ["Perform deletion"],
|
|
85
|
+
test_obligations: [],
|
|
86
|
+
validation_requirements: [],
|
|
87
|
+
traceability_requirements: [],
|
|
88
|
+
risks: [],
|
|
89
|
+
assumptions: [],
|
|
90
|
+
unknowns: [],
|
|
91
|
+
blockers: [],
|
|
92
|
+
created_at: new Date().toISOString(),
|
|
93
|
+
updated_at: new Date().toISOString(),
|
|
94
|
+
snapshot_id: "LATEST"
|
|
95
|
+
};
|
|
96
|
+
const generatedTests = generateTestObligations(planDel);
|
|
97
|
+
console.log("Auto-Generated Test Obligations:");
|
|
98
|
+
generatedTests.forEach(t => console.log("-", t));
|
|
99
|
+
// Missing test obligations validation check
|
|
100
|
+
let errorsDel = validatePlan(ecosystemRoot, planDel);
|
|
101
|
+
console.log("Validation Errors (Missing Tests in Plan):", errorsDel);
|
|
102
|
+
let validationDel = aggregateValidation("CONFIRMED", errorsDel, [], [], [], false);
|
|
103
|
+
console.log("Overall State (Missing Tests):", validationDel.overall);
|
|
104
|
+
// 4. Stale Evidence Check
|
|
105
|
+
console.log("\\n--- STALE EVIDENCE ---");
|
|
106
|
+
planDel.test_obligations = generatedTests;
|
|
107
|
+
planDel.created_at = new Date(Date.now() - 100000).toISOString(); // Make it older than last extraction
|
|
108
|
+
savePlan(ecosystemRoot, planDel);
|
|
109
|
+
let errorsStale = validatePlan(ecosystemRoot, planDel);
|
|
110
|
+
console.log("Validation Errors (Stale Plan):", errorsStale);
|
|
111
|
+
let validationStale = aggregateValidation("CONFIRMED", errorsStale, [], [], [], false);
|
|
112
|
+
console.log("Overall State (Stale):", validationStale.overall);
|
|
113
|
+
// 5. Experience Persistence
|
|
114
|
+
console.log("\\n--- EXPERIENCE PERSISTENCE ---");
|
|
115
|
+
const exp = recordExperience(ecosystemRoot, {
|
|
116
|
+
id: "EXP-1",
|
|
117
|
+
category: "ANTI_PATTERN",
|
|
118
|
+
title: "Silent Deletion",
|
|
119
|
+
summary: "Never delete users without producing a user-deleted event.",
|
|
120
|
+
lesson: "Never delete users without producing a user-deleted event.",
|
|
121
|
+
context: "Discovered during User Deletion Phase 4 benchmark",
|
|
122
|
+
related_artifacts: []
|
|
123
|
+
});
|
|
124
|
+
console.log("Recorded Experience ID:", exp.id);
|
|
125
|
+
const loadedExp = loadExperience(ecosystemRoot);
|
|
126
|
+
console.log("Loaded Experiences:", loadedExp.length);
|
|
127
|
+
console.log("Experience Title:", loadedExp[0].title);
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
runPhase4Benchmark();
|
|
131
|
+
}
|
|
132
|
+
catch (e) {
|
|
133
|
+
console.error("Benchmark failed:", e);
|
|
134
|
+
process.exit(1);
|
|
135
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { savePlan, validatePlan } from "./implementation_plan.js";
|
|
4
|
+
import { generateTestObligations, aggregateValidation } from "./validation_engine.js";
|
|
5
|
+
import { executeTests, ingestTestResults, loadTestResults, correlateTestFailure } from "./test_engine.js";
|
|
6
|
+
import { saveRequirement } from "./requirement_manager.js";
|
|
7
|
+
import { saveTraceability } from "./traceability_engine.js";
|
|
8
|
+
import { performFullExtraction } from "./server.js";
|
|
9
|
+
const ecosystemRoot = "c:\\Users\\Yash\\Desktop\\mcp";
|
|
10
|
+
function setupRequirement(id, title, status, decisions = []) {
|
|
11
|
+
const req = {
|
|
12
|
+
id,
|
|
13
|
+
title,
|
|
14
|
+
status,
|
|
15
|
+
raw_request: title,
|
|
16
|
+
clarification_questions: [],
|
|
17
|
+
decisions
|
|
18
|
+
};
|
|
19
|
+
saveRequirement(ecosystemRoot, req);
|
|
20
|
+
}
|
|
21
|
+
function runPhase5Benchmark() {
|
|
22
|
+
console.log("=== RUNNING PHASE 5 TESTING INTELLIGENCE BENCHMARK ===");
|
|
23
|
+
// Initial Setup
|
|
24
|
+
const stats = performFullExtraction(ecosystemRoot);
|
|
25
|
+
const dnaPath = path.join(ecosystemRoot, ".secufusion", "dna.json");
|
|
26
|
+
const dna = JSON.parse(fs.readFileSync(dnaPath, "utf-8"));
|
|
27
|
+
const graph = dna.graph;
|
|
28
|
+
saveTraceability(ecosystemRoot, []);
|
|
29
|
+
fs.writeFileSync(path.join(ecosystemRoot, '.secufusion', 'tests', 'results.json'), '[]'); // Clear tests
|
|
30
|
+
console.log("\\n--- 1. TEST DISCOVERY & INGESTION (User Deletion) ---");
|
|
31
|
+
setupRequirement("REQ-DEL-5", "User Deletion Tests", "CONFIRMED", ["Authorized to delete user"]);
|
|
32
|
+
// Create Plan
|
|
33
|
+
let planDel = {
|
|
34
|
+
id: "PLAN-DEL-5",
|
|
35
|
+
requirement_id: "REQ-DEL-5",
|
|
36
|
+
status: "DRAFT",
|
|
37
|
+
summary: "Delete user test execution",
|
|
38
|
+
affected_services: ["sfn-iam-api"],
|
|
39
|
+
affected_files: [],
|
|
40
|
+
api_changes: ["DELETE /users/{id}"],
|
|
41
|
+
database_changes: ["DELETE cascade on users table"],
|
|
42
|
+
event_changes: [],
|
|
43
|
+
frontend_changes: [],
|
|
44
|
+
extension_changes: [],
|
|
45
|
+
security_implications: [],
|
|
46
|
+
implementation_steps: ["Perform deletion"],
|
|
47
|
+
test_obligations: ["Test deletion limits"],
|
|
48
|
+
validation_requirements: [],
|
|
49
|
+
traceability_requirements: [],
|
|
50
|
+
risks: [],
|
|
51
|
+
assumptions: [],
|
|
52
|
+
unknowns: [],
|
|
53
|
+
blockers: [],
|
|
54
|
+
created_at: new Date().toISOString(),
|
|
55
|
+
updated_at: new Date().toISOString(),
|
|
56
|
+
snapshot_id: "LATEST"
|
|
57
|
+
};
|
|
58
|
+
const obligations = generateTestObligations(planDel);
|
|
59
|
+
planDel.test_obligations = obligations;
|
|
60
|
+
savePlan(ecosystemRoot, planDel);
|
|
61
|
+
// Missing test execution -> Should be UNKNOWN
|
|
62
|
+
let errors = validatePlan(ecosystemRoot, planDel);
|
|
63
|
+
console.log("Validation Errors:", errors);
|
|
64
|
+
let tests = loadTestResults(ecosystemRoot);
|
|
65
|
+
let validation = aggregateValidation("CONFIRMED", errors, planDel.test_obligations, tests, [], false);
|
|
66
|
+
console.log("Validation State with MISSING test executions:", validation.overall); // Should be UNKNOWN
|
|
67
|
+
// Ingest PASS results
|
|
68
|
+
const passResult = {
|
|
69
|
+
id: "TR-1",
|
|
70
|
+
test_name: "testUserDeletionCascade",
|
|
71
|
+
suite_name: "UserDeletionTest",
|
|
72
|
+
repository: "sfn-iam-api",
|
|
73
|
+
file_path: "UNKNOWN",
|
|
74
|
+
status: "PASS",
|
|
75
|
+
duration_ms: 150,
|
|
76
|
+
exit_code: 0,
|
|
77
|
+
timestamp: new Date().toISOString(),
|
|
78
|
+
execution_identity: "EXEC-1"
|
|
79
|
+
};
|
|
80
|
+
ingestTestResults(ecosystemRoot, [passResult]);
|
|
81
|
+
tests = loadTestResults(ecosystemRoot);
|
|
82
|
+
validation = aggregateValidation("CONFIRMED", errors, planDel.test_obligations, tests, [], false);
|
|
83
|
+
console.log("Validation State with PASS tests:", validation.overall, "(tests=", validation.tests, ", planErrors=", errors, ", testResults=", tests, ")"); // Should be PASS
|
|
84
|
+
console.log("\\n--- 2. FAIL NORMALIZATION & CORRELATION (Retention) ---");
|
|
85
|
+
// Ingest FAIL results
|
|
86
|
+
const failResult = {
|
|
87
|
+
id: "TR-2",
|
|
88
|
+
test_name: "testRetentionPolicyCleanup",
|
|
89
|
+
suite_name: "RetentionJobTest",
|
|
90
|
+
repository: "sfn-events-api",
|
|
91
|
+
file_path: "UNKNOWN",
|
|
92
|
+
status: "FAIL",
|
|
93
|
+
duration_ms: 1200,
|
|
94
|
+
exit_code: 1,
|
|
95
|
+
failure_message: "Expected 0 events, found 5",
|
|
96
|
+
failure_type: "ASSERTION_FAILURE",
|
|
97
|
+
timestamp: new Date().toISOString(),
|
|
98
|
+
execution_identity: "EXEC-2"
|
|
99
|
+
};
|
|
100
|
+
// Map traceability to a fake RetentionJob node
|
|
101
|
+
graph.nodes.push({ id: "CLASS:RetentionJob", type: "CLASS", name: "RetentionJob", repository: "sfn-events-api" });
|
|
102
|
+
const edges = [{ source: "TEST:testRetentionPolicyCleanup", target: "CLASS:RetentionJob", relation: "TESTS", confidence: "DIRECT", extractionMethod: "STATIC" }];
|
|
103
|
+
const correlated = correlateTestFailure(graph, edges, failResult);
|
|
104
|
+
console.log("Failure Correlated Artifacts:", correlated);
|
|
105
|
+
ingestTestResults(ecosystemRoot, [failResult]);
|
|
106
|
+
tests = loadTestResults(ecosystemRoot);
|
|
107
|
+
validation = aggregateValidation("CONFIRMED", errors, planDel.test_obligations, tests, [], false);
|
|
108
|
+
console.log("Validation State with FAIL test:", validation.overall); // Should be FAIL
|
|
109
|
+
console.log("\\n--- 3. TEST EXECUTION (Controlled BOUNDARY) ---");
|
|
110
|
+
// Run an echo command
|
|
111
|
+
const execResults = executeTests(ecosystemRoot, "echo 'Testing controlled execution'", ".");
|
|
112
|
+
console.log("Execution Result Status:", execResults[0].status);
|
|
113
|
+
console.log("Execution Output Captured:", execResults.length);
|
|
114
|
+
}
|
|
115
|
+
try {
|
|
116
|
+
runPhase5Benchmark();
|
|
117
|
+
}
|
|
118
|
+
catch (e) {
|
|
119
|
+
console.error("Benchmark failed:", e);
|
|
120
|
+
process.exit(1);
|
|
121
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { recordExperience, getEngineeringExperience, validateExperience, markExperienceStale, linkExperience, findExperienceConflicts } from "./experience_manager.js";
|
|
4
|
+
function resetExperience(ecosystemRoot) {
|
|
5
|
+
const expPath = path.join(ecosystemRoot, '.secufusion', 'experience.json');
|
|
6
|
+
if (fs.existsSync(expPath))
|
|
7
|
+
fs.unlinkSync(expPath);
|
|
8
|
+
}
|
|
9
|
+
function runBenchmark() {
|
|
10
|
+
console.log("=== RUNNING PHASE 6 ENGINEERING KNOWLEDGE BENCHMARK ===\n");
|
|
11
|
+
// Use a temp workspace to avoid polluting real .secufusion
|
|
12
|
+
const ecosystemRoot = path.join(process.cwd(), '.benchmark_tmp_ph6');
|
|
13
|
+
if (!fs.existsSync(ecosystemRoot))
|
|
14
|
+
fs.mkdirSync(ecosystemRoot, { recursive: true });
|
|
15
|
+
resetExperience(ecosystemRoot);
|
|
16
|
+
try {
|
|
17
|
+
console.log("--- 1. RECORDING PROPOSED EXPERIENCE ---");
|
|
18
|
+
const exp1 = recordExperience(ecosystemRoot, {
|
|
19
|
+
id: "EXP-RETENTION",
|
|
20
|
+
title: "Retention Cleanup Idempotency",
|
|
21
|
+
summary: "Previous retention cleanup required idempotent execution due to retries.",
|
|
22
|
+
category: "SCHEDULING",
|
|
23
|
+
lesson: "Always use idempotent queries for cleanup jobs in RetentionPolicy.",
|
|
24
|
+
outcome: "FAILURE",
|
|
25
|
+
affected_components: ["CLASS:RetentionJob", "CLASS:Scheduler"],
|
|
26
|
+
status: "PROPOSED",
|
|
27
|
+
evidence: "Run failures in Phase 3"
|
|
28
|
+
});
|
|
29
|
+
console.log(`Recorded: ${exp1.id} with status ${exp1.status}`);
|
|
30
|
+
console.log("\n--- 2. VALIDATING EXPERIENCE WITH EVIDENCE ---");
|
|
31
|
+
const validatedExp = validateExperience(ecosystemRoot, exp1.id, "TEST-RESULT: PASS for idempotent query");
|
|
32
|
+
console.log(`Validated: ${validatedExp.id}, Status: ${validatedExp.status}, Evidence: ${validatedExp.evidence}`);
|
|
33
|
+
console.log("\n--- 3. TRACEABILITY INTEGRATION (LINKING) ---");
|
|
34
|
+
const linkedExp = linkExperience(ecosystemRoot, exp1.id, {
|
|
35
|
+
source_requirement_id: "REQ-101",
|
|
36
|
+
source_traceability_ids: ["TR-99"]
|
|
37
|
+
});
|
|
38
|
+
console.log(`Linked: ${linkedExp.id}, Source Req: ${linkedExp.source_requirement_id}`);
|
|
39
|
+
console.log("\n--- 4. RETENTION BENCHMARK (STALENESS) ---");
|
|
40
|
+
// Simulated: We discover that the current repository scheduler implementation has changed
|
|
41
|
+
console.log("Simulating repository evidence check: Scheduler has been replaced with Quartz.");
|
|
42
|
+
const staleExp = markExperienceStale(ecosystemRoot, exp1.id, "Scheduler implementation replaced by Quartz, making old idempotency logic obsolete.");
|
|
43
|
+
console.log(`Stale State: ${staleExp.status}`);
|
|
44
|
+
console.log(`Stale Reason in context: ${staleExp.context}`);
|
|
45
|
+
console.log("\n--- 5. USER DELETION BENCHMARK (CONFLICTS) ---");
|
|
46
|
+
recordExperience(ecosystemRoot, {
|
|
47
|
+
id: "EXP-DEL-1",
|
|
48
|
+
title: "Hard delete user events",
|
|
49
|
+
summary: "User deletion must cascade to events.",
|
|
50
|
+
category: "DELETION",
|
|
51
|
+
lesson: "Use cascade delete.",
|
|
52
|
+
outcome: "SUCCESS",
|
|
53
|
+
affected_components: ["CLASS:UserDeletionService"],
|
|
54
|
+
status: "VALIDATED"
|
|
55
|
+
});
|
|
56
|
+
recordExperience(ecosystemRoot, {
|
|
57
|
+
id: "EXP-DEL-2",
|
|
58
|
+
title: "Anonymize user events",
|
|
59
|
+
summary: "User deletion must anonymize events.",
|
|
60
|
+
category: "DELETION",
|
|
61
|
+
lesson: "Use anonymization strategy.",
|
|
62
|
+
outcome: "SUCCESS",
|
|
63
|
+
affected_components: ["CLASS:UserDeletionService"],
|
|
64
|
+
status: "VALIDATED"
|
|
65
|
+
});
|
|
66
|
+
const conflicts = findExperienceConflicts(ecosystemRoot);
|
|
67
|
+
console.log(`Conflicts Detected: ${conflicts.length} records on shared component.`);
|
|
68
|
+
conflicts.forEach(c => console.log(` - ${c.id}: ${c.title} (Outcome: ${c.outcome})`));
|
|
69
|
+
console.log("\n--- 6. BOUNDED RETRIEVAL ---");
|
|
70
|
+
const retrieved = getEngineeringExperience(ecosystemRoot, "", { affected_components: ["CLASS:RetentionJob"] });
|
|
71
|
+
console.log(`Retrieved ${retrieved.length} records for CLASS:RetentionJob`);
|
|
72
|
+
console.log(`Retrieved Record Status: ${retrieved[0].status}`);
|
|
73
|
+
console.log("\n=== BENCHMARK COMPLETE (ALL PASS) ===");
|
|
74
|
+
}
|
|
75
|
+
catch (e) {
|
|
76
|
+
console.error("Benchmark Failed:", e.message);
|
|
77
|
+
process.exit(1);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
runBenchmark();
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { compareRepositorySnapshots, detectArchitectureDrift, detectRequirementDrift, detectTraceabilityDrift, getRegressionContext } from "./regression_engine.js";
|
|
4
|
+
import { recordExperience } from "./experience_manager.js";
|
|
5
|
+
function reset(ecosystemRoot) {
|
|
6
|
+
if (fs.existsSync(ecosystemRoot)) {
|
|
7
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
8
|
+
}
|
|
9
|
+
fs.mkdirSync(ecosystemRoot, { recursive: true });
|
|
10
|
+
}
|
|
11
|
+
function runBenchmark() {
|
|
12
|
+
console.log("=== RUNNING PHASE 7 CONTINUOUS INTELLIGENCE BENCHMARK ===\n");
|
|
13
|
+
const ecosystemRoot = path.join(process.cwd(), '.benchmark_tmp_ph7');
|
|
14
|
+
reset(ecosystemRoot);
|
|
15
|
+
try {
|
|
16
|
+
// --- Setup Data ---
|
|
17
|
+
const baseSnapshot = {
|
|
18
|
+
schema_version: "1.0",
|
|
19
|
+
nodes: [
|
|
20
|
+
{ id: "SERVICE:sfn-iam-api", type: "SERVICE", name: "sfn-iam-api", repository: "sfn-iam-api" },
|
|
21
|
+
{ id: "SERVICE:sfn-tenants-api", type: "SERVICE", name: "sfn-tenants-api", repository: "sfn-tenants-api" },
|
|
22
|
+
{ id: "CLASS:UserDeletionService", type: "CLASS", name: "UserDeletionService", repository: "sfn-iam-api" },
|
|
23
|
+
{ id: "CLASS:RetentionJob", type: "CLASS", name: "RetentionJob", repository: "sfn-tenants-api" },
|
|
24
|
+
{ id: "API:GET:/api/tenants", type: "API", name: "GET /api/tenants", repository: "sfn-tenants-api" },
|
|
25
|
+
{ id: "TOPIC:user-deleted", type: "TOPIC", name: "user-deleted" }
|
|
26
|
+
],
|
|
27
|
+
edges: [
|
|
28
|
+
{ id: "E1", source: "CLASS:UserDeletionService", target: "TOPIC:user-deleted", type: "PRODUCES" },
|
|
29
|
+
{ id: "E2", source: "SERVICE:sfn-iam-api", target: "SERVICE:sfn-tenants-api", type: "CALLS" }
|
|
30
|
+
]
|
|
31
|
+
};
|
|
32
|
+
const currentSnapshot = {
|
|
33
|
+
schema_version: "1.0",
|
|
34
|
+
nodes: [
|
|
35
|
+
{ id: "SERVICE:sfn-iam-api", type: "SERVICE", name: "sfn-iam-api", repository: "sfn-iam-api" },
|
|
36
|
+
{ id: "SERVICE:sfn-tenants-api", type: "SERVICE", name: "sfn-tenants-api", repository: "sfn-tenants-api" },
|
|
37
|
+
// UserDeletionService still exists but metadata/logic changed (represented via snapshot diff)
|
|
38
|
+
{ id: "CLASS:UserDeletionService", type: "CLASS", name: "UserDeletionService", repository: "sfn-iam-api", metadata: { updated: true } },
|
|
39
|
+
{ id: "CLASS:RetentionJob", type: "CLASS", name: "RetentionJob", repository: "sfn-tenants-api" },
|
|
40
|
+
// API GET /api/tenants is REMOVED
|
|
41
|
+
{ id: "TOPIC:user-deleted", type: "TOPIC", name: "user-deleted" }
|
|
42
|
+
],
|
|
43
|
+
edges: [
|
|
44
|
+
// Produces edge E1 is REMOVED (Architecture drift)
|
|
45
|
+
// Calls edge E2 is REMOVED
|
|
46
|
+
]
|
|
47
|
+
};
|
|
48
|
+
const reqs = [
|
|
49
|
+
{
|
|
50
|
+
id: "REQ-1", status: "CONFIRMED",
|
|
51
|
+
apis: ["GET:/api/tenants"]
|
|
52
|
+
}
|
|
53
|
+
];
|
|
54
|
+
const links = [
|
|
55
|
+
{ id: "L1", source: "REQ-1", target: "API:GET:/api/tenants", type: "IMPLEMENTS", provenance: "USER_CONFIRMED", confidence: "CONFIRMED" }
|
|
56
|
+
];
|
|
57
|
+
const allTests = [
|
|
58
|
+
{ id: "TEST-1", name: "testUserDeletionCascade", target_components: ["CLASS:UserDeletionService"] }
|
|
59
|
+
];
|
|
60
|
+
recordExperience(ecosystemRoot, {
|
|
61
|
+
id: "EXP-1", title: "Orphaned Events", summary: "User deletion left event references orphaned.",
|
|
62
|
+
category: "DELETION", lesson: "Validate cascade deletion.", outcome: "FAILURE",
|
|
63
|
+
affected_components: ["CLASS:UserDeletionService"], status: "VALIDATED"
|
|
64
|
+
});
|
|
65
|
+
// --- Benchmark 1: Snapshot Comparison ---
|
|
66
|
+
console.log("--- 1. SNAPSHOT COMPARISON ---");
|
|
67
|
+
const changes = compareRepositorySnapshots(baseSnapshot, currentSnapshot);
|
|
68
|
+
console.log(`Changes detected: ${changes.length}`);
|
|
69
|
+
changes.forEach(c => console.log(` - ${c.category}: ${c.artifact_id}`));
|
|
70
|
+
// --- Benchmark 2: Architecture Drift ---
|
|
71
|
+
console.log("\n--- 2. ARCHITECTURE DRIFT (BENCHMARK #26) ---");
|
|
72
|
+
const archDrift = detectArchitectureDrift(baseSnapshot, currentSnapshot);
|
|
73
|
+
console.log(`Drifts detected: ${archDrift.length}`);
|
|
74
|
+
archDrift.forEach(d => console.log(` - ${d.category} on ${d.affected_component} (Status: ${d.status})`));
|
|
75
|
+
// --- Benchmark 3: Requirement Drift ---
|
|
76
|
+
console.log("\n--- 3. REQUIREMENT DRIFT (BENCHMARK #27) ---");
|
|
77
|
+
const reqDrift = detectRequirementDrift(reqs, currentSnapshot);
|
|
78
|
+
console.log(`Requirement Drifts: ${reqDrift.length}`);
|
|
79
|
+
reqDrift.forEach(d => console.log(` - ${d.condition}: ${d.expected_artifact} for ${d.requirement_id}`));
|
|
80
|
+
// --- Benchmark 4: Traceability Drift ---
|
|
81
|
+
console.log("\n--- 4. TRACEABILITY DRIFT ---");
|
|
82
|
+
const traceDrift = detectTraceabilityDrift(links, currentSnapshot);
|
|
83
|
+
console.log(`Traceability Drifts: ${traceDrift.length}`);
|
|
84
|
+
traceDrift.forEach(d => console.log(` - ${d.condition}: ${d.target_id}`));
|
|
85
|
+
// --- Benchmark 5: Regression Tests & Context ---
|
|
86
|
+
console.log("\n--- 5. USER DELETION REGRESSION (BENCHMARK #25) & COVERAGE (BENCHMARK #28) ---");
|
|
87
|
+
const context = getRegressionContext(ecosystemRoot, baseSnapshot, currentSnapshot, reqs, links, allTests);
|
|
88
|
+
console.log(`Historical Failures matching changes: ${context.historical_failures.length}`);
|
|
89
|
+
console.log(`Regression Tests found: ${context.regression_tests.length}`);
|
|
90
|
+
console.log("\n--- 6. RISK INDICATORS (BENCHMARK #29 CROSS-SERVICE) ---");
|
|
91
|
+
context.risk_indicators.forEach(r => {
|
|
92
|
+
console.log(` - ${r.indicator} (${r.risk_level})`);
|
|
93
|
+
if (r.indicator === "CROSS_SERVICE_CHANGE")
|
|
94
|
+
console.log(` Services: ${r.evidence.services.join(", ")}`);
|
|
95
|
+
});
|
|
96
|
+
console.log("\n=== BENCHMARK COMPLETE (ALL PASS) ===");
|
|
97
|
+
}
|
|
98
|
+
catch (e) {
|
|
99
|
+
console.error("Benchmark Failed:", e.message);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
runBenchmark();
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { discoverUIPatterns, generateStaticUIReference, getDesignContext } from "./ui_intelligence.js";
|
|
4
|
+
function reset(ecosystemRoot) {
|
|
5
|
+
if (fs.existsSync(ecosystemRoot)) {
|
|
6
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
7
|
+
}
|
|
8
|
+
fs.mkdirSync(ecosystemRoot, { recursive: true });
|
|
9
|
+
}
|
|
10
|
+
function runBenchmark() {
|
|
11
|
+
console.log("=== RUNNING PHASE 8 UX & UI DESIGN BENCHMARK ===\n");
|
|
12
|
+
const ecosystemRoot = path.join(process.cwd(), '.benchmark_tmp_ph8');
|
|
13
|
+
reset(ecosystemRoot);
|
|
14
|
+
try {
|
|
15
|
+
// --- Setup Mock State ---
|
|
16
|
+
const snapshot = {
|
|
17
|
+
schema_version: "1.0",
|
|
18
|
+
nodes: [
|
|
19
|
+
{ id: "FRONTEND_APP:sfn-web-ui", type: "FRONTEND_APP", name: "sfn-web-ui" },
|
|
20
|
+
{ id: "UI_PATTERN:TenantSelector", type: "UI_PATTERN", name: "TenantSelector", repository: "sfn-web-ui", evidence: { filePath: "src/components/TenantSelector.tsx" }, metadata: { type: "React", description: "Standard tenant dropdown" } },
|
|
21
|
+
{ id: "UI_PATTERN:ConfirmationModal", type: "UI_PATTERN", name: "ConfirmationModal", repository: "sfn-web-ui", evidence: { filePath: "src/components/ConfirmationModal.tsx" }, metadata: { type: "React", description: "Destructive confirmation dialog" } },
|
|
22
|
+
{ id: "FRONTEND_COMPONENT:RetentionSettings", type: "FRONTEND_COMPONENT", name: "RetentionSettings", repository: "sfn-web-ui", evidence: { filePath: "src/pages/RetentionSettings.tsx" }, metadata: { type: "React" } },
|
|
23
|
+
{ id: "API:DELETE:/api/users", type: "API", name: "DELETE /api/users", repository: "sfn-iam-api" }
|
|
24
|
+
],
|
|
25
|
+
edges: []
|
|
26
|
+
};
|
|
27
|
+
fs.mkdirSync(path.join(ecosystemRoot, ".secufusion", "requirements"), { recursive: true });
|
|
28
|
+
const retentionReq = { id: "REQ-RETENTION", status: "CONFIRMED" };
|
|
29
|
+
const deletionReq = { id: "REQ-DELETE-USER", status: "CONFIRMED", apis: ["DELETE:/api/users"] };
|
|
30
|
+
fs.writeFileSync(path.join(ecosystemRoot, ".secufusion", "requirements", "REQ-RETENTION.json"), JSON.stringify(retentionReq));
|
|
31
|
+
fs.writeFileSync(path.join(ecosystemRoot, ".secufusion", "requirements", "REQ-DELETE-USER.json"), JSON.stringify(deletionReq));
|
|
32
|
+
// --- Benchmark 1: Retention Policy (New Feature) ---
|
|
33
|
+
console.log("--- 1. RETENTION POLICY BENCHMARK ---");
|
|
34
|
+
const retentionContext = getDesignContext(ecosystemRoot, "REQ-RETENTION", snapshot);
|
|
35
|
+
console.log(`Context status: ${retentionContext.status}`);
|
|
36
|
+
const tenantPatterns = discoverUIPatterns(ecosystemRoot, "tenant selector", snapshot);
|
|
37
|
+
console.log(`Discovered Tenant Patterns: ${tenantPatterns.length}`);
|
|
38
|
+
tenantPatterns.forEach(p => console.log(` - ${p.name} (Source: ${p.evidence.location.filePath})`));
|
|
39
|
+
const retentionHtml = `<html><body><h1>Retention Policy</h1><p>Set retention here</p></body></html>`;
|
|
40
|
+
const retentionPath = generateStaticUIReference(ecosystemRoot, "DESIGN-RETENTION", retentionHtml, { status: "DRAFT" });
|
|
41
|
+
console.log(`Generated HTML at: ${retentionPath}`);
|
|
42
|
+
const retentionContent = fs.readFileSync(retentionPath, "utf-8");
|
|
43
|
+
if (retentionContent.includes("REFERENCE_ONLY"))
|
|
44
|
+
console.log("SUCCESS: Retention HTML contains REFERENCE_ONLY marker.");
|
|
45
|
+
// --- Benchmark 2: User Deletion (Destructive Operation) ---
|
|
46
|
+
console.log("\n--- 2. USER DELETION BENCHMARK ---");
|
|
47
|
+
const deletionContext = getDesignContext(ecosystemRoot, "REQ-DELETE-USER", snapshot);
|
|
48
|
+
console.log(`Architecture Context Nodes: ${deletionContext.architecture_context.length}`);
|
|
49
|
+
deletionContext.architecture_context.forEach((n) => console.log(` - Affected: ${n.id}`));
|
|
50
|
+
const deletePatterns = discoverUIPatterns(ecosystemRoot, "confirmation destructive", snapshot);
|
|
51
|
+
console.log(`Discovered Confirmation Patterns: ${deletePatterns.length}`);
|
|
52
|
+
deletePatterns.forEach(p => console.log(` - ${p.name} (Source: ${p.evidence.location.filePath})`));
|
|
53
|
+
const deletionHtml = `<html><body><h1>Delete User</h1></body></html>`;
|
|
54
|
+
const deletionPath = generateStaticUIReference(ecosystemRoot, "DESIGN-DELETE-USER", deletionHtml, { status: "USER_REVIEW_REQUIRED" });
|
|
55
|
+
console.log(`Generated HTML at: ${deletionPath}`);
|
|
56
|
+
const deletionContent = fs.readFileSync(deletionPath, "utf-8");
|
|
57
|
+
if (deletionContent.includes("REFERENCE_ONLY"))
|
|
58
|
+
console.log("SUCCESS: Deletion HTML contains REFERENCE_ONLY marker.");
|
|
59
|
+
// --- Benchmark 3: Validate Metadata ---
|
|
60
|
+
console.log("\n--- 3. DESIGN METADATA VALIDATION ---");
|
|
61
|
+
const metaPath = path.join(ecosystemRoot, ".secufusion", "designs", "DESIGN-DELETE-USER", "design.json");
|
|
62
|
+
const meta = JSON.parse(fs.readFileSync(metaPath, "utf-8"));
|
|
63
|
+
console.log(`Design Status: ${meta.status}`);
|
|
64
|
+
console.log(`Design ID: ${meta.design_id}`);
|
|
65
|
+
console.log("\n=== BENCHMARK COMPLETE (ALL PASS) ===");
|
|
66
|
+
}
|
|
67
|
+
catch (e) {
|
|
68
|
+
console.error("Benchmark Failed:", e.message);
|
|
69
|
+
process.exit(1);
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
runBenchmark();
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { inspectWorkspaceRepositories, getEvidenceCompleteness } from "./workspace_intelligence.js";
|
|
4
|
+
import { performFullExtraction } from "./server.js";
|
|
5
|
+
function reset(ecosystemRoot) {
|
|
6
|
+
if (fs.existsSync(ecosystemRoot)) {
|
|
7
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
8
|
+
}
|
|
9
|
+
fs.mkdirSync(ecosystemRoot, { recursive: true });
|
|
10
|
+
}
|
|
11
|
+
function runBenchmark() {
|
|
12
|
+
console.log("=== RUNNING PHASE 9 WORKSPACE INTELLIGENCE BENCHMARK ===\n");
|
|
13
|
+
const ecosystemRoot = path.join(process.cwd(), '.benchmark_tmp_ph9');
|
|
14
|
+
reset(ecosystemRoot);
|
|
15
|
+
try {
|
|
16
|
+
// --- Setup Mock Workspace ---
|
|
17
|
+
const coreRepos = ["sfn-iam-api", "sfn-events-api", "sfn-tenants-api", "sfn-web-ui"];
|
|
18
|
+
const optionalRepos = ["sfn-auth-api", "sfn-browser-ext"];
|
|
19
|
+
[...coreRepos, ...optionalRepos].forEach(repo => {
|
|
20
|
+
fs.mkdirSync(path.join(ecosystemRoot, repo), { recursive: true });
|
|
21
|
+
fs.writeFileSync(path.join(ecosystemRoot, repo, "index.ts"), "console.log('test');");
|
|
22
|
+
});
|
|
23
|
+
// --- Benchmark 1: All Available ---
|
|
24
|
+
console.log("--- 1. FULL WORKSPACE BENCHMARK ---");
|
|
25
|
+
let inspection = inspectWorkspaceRepositories(ecosystemRoot);
|
|
26
|
+
console.log(`Status: ${inspection.status}`);
|
|
27
|
+
if (inspection.status !== "READY")
|
|
28
|
+
throw new Error("Status should be READY");
|
|
29
|
+
if (inspection.core_found.length !== 4)
|
|
30
|
+
throw new Error("Should find 4 core repos");
|
|
31
|
+
let completeness = getEvidenceCompleteness(ecosystemRoot);
|
|
32
|
+
console.log(`Completeness Status: ${completeness.completeness_status}`);
|
|
33
|
+
// --- Benchmark 2: Missing Optional Repository ---
|
|
34
|
+
console.log("\n--- 2. MISSING OPTIONAL BENCHMARK ---");
|
|
35
|
+
fs.rmSync(path.join(ecosystemRoot, "sfn-browser-ext"), { recursive: true, force: true });
|
|
36
|
+
inspection = inspectWorkspaceRepositories(ecosystemRoot);
|
|
37
|
+
console.log(`Status after missing optional: ${inspection.status}`);
|
|
38
|
+
if (inspection.status !== "READY")
|
|
39
|
+
throw new Error("Status should still be READY");
|
|
40
|
+
if (inspection.optional_missing.includes("sfn-browser-ext") === false)
|
|
41
|
+
throw new Error("Missing optional should be tracked");
|
|
42
|
+
// --- Benchmark 3: Missing Core Repository ---
|
|
43
|
+
console.log("\n--- 3. MISSING CORE BENCHMARK ---");
|
|
44
|
+
fs.rmSync(path.join(ecosystemRoot, "sfn-iam-api"), { recursive: true, force: true });
|
|
45
|
+
inspection = inspectWorkspaceRepositories(ecosystemRoot);
|
|
46
|
+
console.log(`Status after missing core: ${inspection.status}`);
|
|
47
|
+
if (inspection.status !== "PARTIAL")
|
|
48
|
+
throw new Error("Status should be PARTIAL (some core found, some missing)");
|
|
49
|
+
if (inspection.core_missing.includes("sfn-iam-api") === false)
|
|
50
|
+
throw new Error("Missing core should be tracked");
|
|
51
|
+
completeness = getEvidenceCompleteness(ecosystemRoot);
|
|
52
|
+
console.log(`Completeness Message: ${completeness.message}`);
|
|
53
|
+
// --- Benchmark 4: Extraction Respects Boundaries ---
|
|
54
|
+
console.log("\n--- 4. EXTRACTION BOUNDARY BENCHMARK ---");
|
|
55
|
+
const stats = performFullExtraction(ecosystemRoot);
|
|
56
|
+
console.log(`Extraction Status: ${stats.inspectionStatus}`);
|
|
57
|
+
if (stats.inspectionStatus !== "PARTIAL")
|
|
58
|
+
throw new Error("Extraction should bubble up PARTIAL status");
|
|
59
|
+
console.log("\n=== BENCHMARK COMPLETE (ALL PASS) ===");
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
console.error("Benchmark Failed:", e.message);
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
fs.rmSync(ecosystemRoot, { recursive: true, force: true });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
runBenchmark();
|