vaspera 2.5.0

package/dist/eval/harness.test.js

@@ -0,0 +1,314 @@
+/**
+ * Tests for evaluation harness
+ */
+import { describe, it, expect } from "vitest";
+import { matchFindings } from "./harness.js";
+describe("harness", () => {
+    describe("matchFindings", () => {
+        it("matches findings with exact file and line", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "sql-injection",
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "SQL injection vulnerability",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "sql-injection",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(1);
+            expect(result.falsePositives.length).toBe(0);
+            expect(result.falseNegatives.length).toBe(0);
+        });
+        it("matches findings within 5 line tolerance", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "xss",
+                    file: "render.ts",
+                    line: 12,
+                    severity: "high",
+                    message: "XSS vulnerability",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "render.ts",
+                    line: 10,
+                    severity: "high",
+                    category: "xss",
+                    description: "XSS vulnerability",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(1);
+        });
+        it("does not match findings beyond 5 line tolerance", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "xss",
+                    file: "render.ts",
+                    line: 20,
+                    severity: "high",
+                    message: "XSS vulnerability",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "render.ts",
+                    line: 10,
+                    severity: "high",
+                    category: "xss",
+                    description: "XSS vulnerability",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(0);
+            expect(result.falsePositives.length).toBe(1);
+            expect(result.falseNegatives.length).toBe(1);
+        });
+        it("handles case-insensitive file matching", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "sqli",
+                    file: "./DB.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "sql injection found",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "sql-injection",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(1);
+        });
+        it("identifies false positives", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "false-alarm",
+                    file: "safe.ts",
+                    line: 5,
+                    severity: "high",
+                    message: "Potential issue",
+                    confidence: 100,
+                },
+            ];
+            const expected = [];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(0);
+            expect(result.falsePositives.length).toBe(1);
+            expect(result.falseNegatives.length).toBe(0);
+        });
+        it("identifies false negatives", () => {
+            const actual = [];
+            const expected = [
+                {
+                    file: "vulnerable.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "sql-injection",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(0);
+            expect(result.falsePositives.length).toBe(0);
+            expect(result.falseNegatives.length).toBe(1);
+        });
+        it("ignores non-vulnerable expected findings", () => {
+            const actual = [];
+            const expected = [
+                {
+                    file: "safe.ts",
+                    line: 10,
+                    severity: "low",
+                    category: "sql-injection",
+                    description: "Safe parameterized query",
+                    isVulnerable: false,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            // Non-vulnerable findings should not be counted as false negatives
+            expect(result.falseNegatives.length).toBe(0);
+        });
+        it("handles multiple findings correctly", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "sqli",
+                    file: "a.ts",
+                    line: 5,
+                    severity: "critical",
+                    message: "sql injection",
+                    confidence: 100,
+                },
+                {
+                    scanner: "gitleaks",
+                    ruleId: "aws-key",
+                    file: "b.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "AWS key found",
+                    confidence: 100,
+                },
+                {
+                    scanner: "semgrep",
+                    ruleId: "unused",
+                    file: "c.ts",
+                    line: 15,
+                    severity: "low",
+                    message: "Unused variable",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "a.ts",
+                    line: 5,
+                    severity: "critical",
+                    category: "sql-injection",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+                {
+                    file: "b.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "secrets",
+                    description: "Hardcoded secret",
+                    isVulnerable: true,
+                },
+                {
+                    file: "d.ts",
+                    line: 20,
+                    severity: "high",
+                    category: "xss",
+                    description: "Missed XSS",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(2); // a.ts and b.ts matched
+            expect(result.falsePositives.length).toBe(1); // c.ts is a false positive
+            expect(result.falseNegatives.length).toBe(1); // d.ts was missed
+        });
+        it("matches based on CWE ID", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "CWE-89-sql",
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "Potential SQL issue",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "command-injection", // Different category
+                    cweId: "CWE-89",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(1);
+        });
+        it("handles relative path matching", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "sqli",
+                    file: "/tmp/project/src/db.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "SQL injection",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "sql-injection",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            expect(result.truePositives.length).toBe(1);
+        });
+        it("only matches each finding once", () => {
+            const actual = [
+                {
+                    scanner: "semgrep",
+                    ruleId: "sqli-1",
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "SQL injection pattern 1",
+                    confidence: 100,
+                },
+                {
+                    scanner: "semgrep",
+                    ruleId: "sqli-2",
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    message: "SQL injection pattern 2",
+                    confidence: 100,
+                },
+            ];
+            const expected = [
+                {
+                    file: "db.ts",
+                    line: 10,
+                    severity: "critical",
+                    category: "sql-injection",
+                    description: "SQL injection",
+                    isVulnerable: true,
+                },
+            ];
+            const result = matchFindings(actual, expected);
+            // Only one should match, the other is a false positive
+            expect(result.truePositives.length).toBe(1);
+            expect(result.falsePositives.length).toBe(1);
+            expect(result.falseNegatives.length).toBe(0);
+        });
+    });
+});
+//# sourceMappingURL=harness.test.js.map

package/dist/eval/harness.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.test.js","sourceRoot":"","sources":["../../src/eval/harness.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAG7C,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACvB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,eAAe;oBACvB,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,6BAA6B;oBACtC,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,MAAM;oBAChB,OAAO,EAAE,mBAAmB;oBAC5B,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,MAAM;oBAChB,QAAQ,EAAE,KAAK;oBACf,WAAW,EAAE,mBAAmB;oBAChC,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,KAAK;oBACb,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,MAAM;oBAChB,OAAO,EAAE,mBAAmB;oBAC5B,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,WAAW;oBACjB,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,MAAM;oBAChB,QAAQ,EAAE,KAAK;oBACf,WAAW,EAAE,mBAAmB;oBAChC,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,MAAM;oBACd,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,qBAAqB;oBAC9B,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,CAAC;oBACP,QAAQ,EAAE,MAAM;oBAChB,OAAO,EAAE,iBAAiB;oBAC1B,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB,EAAE,CAAC;YAEvC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,MAAM,GAAoB,EAAE,CAAC;YAEnC,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAoB,EAAE,CAAC;YAEnC,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,KAAK;oBACf,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,0BAA0B;oBACvC,YAAY,EAAE,KAAK;iBACpB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,mEAAmE;YACnE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,MAAM;oBACd,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,CAAC;oBACP,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,eAAe;oBACxB,UAAU,EAAE,GAAG;iBAChB;gBACD;oBACE,OAAO,EAAE,UAAU;oBACnB,MAAM,EAAE,SAAS;oBACjB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,eAAe;oBACxB,UAAU,EAAE,GAAG;iBAChB;gBACD;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,QAAQ;oBAChB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,KAAK;oBACf,OAAO,EAAE,iBAAiB;oBAC1B,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,CAAC;oBACP,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,SAAS;oBACnB,WAAW,EAAE,kBAAkB;oBAC/B,YAAY,EAAE,IAAI;iBACnB;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,MAAM;oBAChB,QAAQ,EAAE,KAAK;oBACf,WAAW,EAAE,YAAY;oBACzB,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;YACrE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,2BAA2B;YACzE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,YAAY;oBACpB,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,qBAAqB;oBAC9B,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,mBAAmB,EAAE,qBAAqB;oBACpD,KAAK,EAAE,QAAQ;oBACf,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,MAAM;oBACd,IAAI,EAAE,wBAAwB;oBAC9B,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,eAAe;oBACxB,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAoB;gBAC9B;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,QAAQ;oBAChB,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,yBAAyB;oBAClC,UAAU,EAAE,GAAG;iBAChB;gBACD;oBACE,OAAO,EAAE,SAAS;oBAClB,MAAM,EAAE,QAAQ;oBAChB,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,OAAO,EAAE,yBAAyB;oBAClC,UAAU,EAAE,GAAG;iBAChB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAsB;gBAClC;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,EAAE;oBACR,QAAQ,EAAE,UAAU;oBACpB,QAAQ,EAAE,eAAe;oBACzB,WAAW,EAAE,eAAe;oBAC5B,YAAY,EAAE,IAAI;iBACnB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAE/C,uDAAuD;YACvD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Evaluation Module
+ *
+ * Exports all evaluation-related functionality for measuring
+ * scanner precision, recall, and stability.
+ *
+ * @module eval
+ */
+export type { TestFixture, FixtureFile, ExpectedFinding, VulnerabilityCategory, FixtureResult, ActualFinding, MatchedFinding, EvaluationMetrics, CategoryMetrics, ScannerMetrics, StabilityMetrics, EvaluationReport, EvaluationConfig, } from "./types.js";
+export { DEFAULT_EVAL_CONFIG, TARGET_METRICS } from "./types.js";
+export { ALL_FIXTURES, sqlInjectionFixtures, xssFixtures, secretsFixtures, typeSafetyFixtures, authFixtures, pathTraversalFixtures, getFixturesByCategory, getFixturesByTag, getFixtureById, getFixtureStats, } from "./fixtures.js";
+export { evaluateFixture, matchFindings, runEvaluation, runStabilityTest, } from "./harness.js";
+export { calculatePrecision, calculateRecall, calculateF1Score, calculateMetrics, calculateStabilityMetrics, meetsTargets, formatPercent, getGrade, } from "./metrics.js";
+export { generateEvaluationReport, formatReportAsMarkdown, generateCompactSummary, generateJsonReport, generateReadmeBadges, } from "./report.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,YAAY,EACV,WAAW,EACX,WAAW,EACX,eAAe,EACf,qBAAqB,EACrB,aAAa,EACb,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGjE,OAAO,EACL,YAAY,EACZ,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,qBAAqB,EACrB,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,eAAe,EACf,aAAa,EACb,aAAa,EACb,gBAAgB,GACjB,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,yBAAyB,EACzB,YAAY,EACZ,aAAa,EACb,QAAQ,GACT,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}

package/dist/eval/index.js

@@ -0,0 +1,18 @@
+/**
+ * Evaluation Module
+ *
+ * Exports all evaluation-related functionality for measuring
+ * scanner precision, recall, and stability.
+ *
+ * @module eval
+ */
+export { DEFAULT_EVAL_CONFIG, TARGET_METRICS } from "./types.js";
+// Fixtures
+export { ALL_FIXTURES, sqlInjectionFixtures, xssFixtures, secretsFixtures, typeSafetyFixtures, authFixtures, pathTraversalFixtures, getFixturesByCategory, getFixturesByTag, getFixtureById, getFixtureStats, } from "./fixtures.js";
+// Harness
+export { evaluateFixture, matchFindings, runEvaluation, runStabilityTest, } from "./harness.js";
+// Metrics
+export { calculatePrecision, calculateRecall, calculateF1Score, calculateMetrics, calculateStabilityMetrics, meetsTargets, formatPercent, getGrade, } from "./metrics.js";
+// Report
+export { generateEvaluationReport, formatReportAsMarkdown, generateCompactSummary, generateJsonReport, generateReadmeBadges, } from "./report.js";
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAmBH,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjE,WAAW;AACX,OAAO,EACL,YAAY,EACZ,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,qBAAqB,EACrB,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AAEvB,UAAU;AACV,OAAO,EACL,eAAe,EACf,aAAa,EACb,aAAa,EACb,gBAAgB,GACjB,MAAM,cAAc,CAAC;AAEtB,UAAU;AACV,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,yBAAyB,EACzB,YAAY,EACZ,aAAa,EACb,QAAQ,GACT,MAAM,cAAc,CAAC;AAEtB,SAAS;AACT,OAAO,EACL,wBAAwB,EACxB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,aAAa,CAAC"}

package/dist/eval/metrics.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Evaluation Metrics
+ *
+ * Calculates precision, recall, F1 score, and other metrics
+ * from evaluation results.
+ *
+ * @module eval/metrics
+ */
+import type { FixtureResult, EvaluationMetrics, StabilityMetrics } from "./types.js";
+/**
+ * Calculate precision from true positives and false positives
+ * Precision = TP / (TP + FP)
+ */
+export declare function calculatePrecision(truePositives: number, falsePositives: number): number;
+/**
+ * Calculate recall from true positives and false negatives
+ * Recall = TP / (TP + FN)
+ */
+export declare function calculateRecall(truePositives: number, falseNegatives: number): number;
+/**
+ * Calculate F1 score from precision and recall
+ * F1 = 2 * (precision * recall) / (precision + recall)
+ */
+export declare function calculateF1Score(precision: number, recall: number): number;
+/**
+ * Calculate aggregate metrics from fixture results
+ */
+export declare function calculateMetrics(results: FixtureResult[]): EvaluationMetrics;
+/**
+ * Calculate stability metrics from multiple runs
+ */
+export declare function calculateStabilityMetrics(allRuns: FixtureResult[][]): StabilityMetrics;
+/**
+ * Check if metrics meet target thresholds
+ */
+export declare function meetsTargets(metrics: EvaluationMetrics, targets?: {
+    precision?: number;
+    recall?: number;
+    f1Score?: number;
+}): {
+    meetsAll: boolean;
+    results: Record<string, {
+        target: number;
+        actual: number;
+        met: boolean;
+    }>;
+};
+/**
+ * Format percentage for display
+ */
+export declare function formatPercent(value: number, decimals?: number): string;
+/**
+ * Get grade based on score
+ */
+export declare function getGrade(score: number): string;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/eval/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,iBAAiB,EAGjB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAEpB;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,GACrB,MAAM,CAIR;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,GACrB,MAAM,CAIR;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAG1E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAiJ5E;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,aAAa,EAAE,EAAE,GACzB,gBAAgB,CAsElB;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE;IACP,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CACb,GACL;IACD,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CAC3E,CA2CA;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,SAAI,GAAG,MAAM,CAEjE;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAS9C"}