vaspera 2.5.0

package/dist/eval/fixtures.test.js

@@ -0,0 +1,193 @@
+/**
+ * Tests for evaluation fixtures
+ */
+import { describe, it, expect } from "vitest";
+import { ALL_FIXTURES, sqlInjectionFixtures, xssFixtures, secretsFixtures, typeSafetyFixtures, authFixtures, pathTraversalFixtures, getFixturesByCategory, getFixturesByTag, getFixtureById, getFixtureStats, } from "./fixtures.js";
+describe("fixtures", () => {
+    describe("ALL_FIXTURES", () => {
+        it("contains all fixture categories", () => {
+            expect(ALL_FIXTURES.length).toBeGreaterThan(0);
+            const categories = new Set(ALL_FIXTURES.map((f) => f.category));
+            expect(categories.has("sql-injection")).toBe(true);
+            expect(categories.has("xss")).toBe(true);
+            expect(categories.has("secrets")).toBe(true);
+            expect(categories.has("type-safety")).toBe(true);
+            expect(categories.has("auth-bypass")).toBe(true);
+            expect(categories.has("path-traversal")).toBe(true);
+        });
+        it("has unique fixture IDs", () => {
+            const ids = ALL_FIXTURES.map((f) => f.id);
+            const uniqueIds = new Set(ids);
+            expect(uniqueIds.size).toBe(ids.length);
+        });
+        it("all fixtures have required fields", () => {
+            for (const fixture of ALL_FIXTURES) {
+                expect(fixture.id).toBeDefined();
+                expect(fixture.name).toBeDefined();
+                expect(fixture.description).toBeDefined();
+                expect(fixture.category).toBeDefined();
+                expect(fixture.files).toBeDefined();
+                expect(fixture.files.length).toBeGreaterThan(0);
+                expect(fixture.expectedFindings).toBeDefined();
+            }
+        });
+        it("all expected findings have required fields", () => {
+            for (const fixture of ALL_FIXTURES) {
+                for (const finding of fixture.expectedFindings) {
+                    expect(finding.file).toBeDefined();
+                    expect(finding.line).toBeDefined();
+                    expect(finding.line).toBeGreaterThan(0);
+                    expect(finding.severity).toBeDefined();
+                    expect(finding.category).toBeDefined();
+                    expect(finding.description).toBeDefined();
+                    expect(typeof finding.isVulnerable).toBe("boolean");
+                }
+            }
+        });
+    });
+    describe("sqlInjectionFixtures", () => {
+        it("contains SQL injection test cases", () => {
+            expect(sqlInjectionFixtures.length).toBeGreaterThan(0);
+            expect(sqlInjectionFixtures.every((f) => f.category === "sql-injection")).toBe(true);
+        });
+        it("includes string concatenation test case", () => {
+            const concatFixture = sqlInjectionFixtures.find((f) => f.id === "sql-001");
+            expect(concatFixture).toBeDefined();
+            expect(concatFixture.name).toContain("String Concatenation");
+        });
+        it("includes template literal test case", () => {
+            const templateFixture = sqlInjectionFixtures.find((f) => f.id === "sql-002");
+            expect(templateFixture).toBeDefined();
+            expect(templateFixture.name).toContain("Template Literal");
+        });
+    });
+    describe("xssFixtures", () => {
+        it("contains XSS test cases", () => {
+            expect(xssFixtures.length).toBeGreaterThan(0);
+            expect(xssFixtures.every((f) => f.category === "xss")).toBe(true);
+        });
+        it("includes React dangerouslySetInnerHTML test case", () => {
+            const reactFixture = xssFixtures.find((f) => f.id === "xss-001");
+            expect(reactFixture).toBeDefined();
+            expect(reactFixture.files[0].language).toBe("tsx");
+        });
+        it("includes DOM manipulation test case", () => {
+            const domFixture = xssFixtures.find((f) => f.id === "xss-002");
+            expect(domFixture).toBeDefined();
+            expect(domFixture.files[0].content).toContain("innerHTML");
+        });
+    });
+    describe("secretsFixtures", () => {
+        it("contains secrets test cases", () => {
+            expect(secretsFixtures.length).toBeGreaterThan(0);
+            expect(secretsFixtures.every((f) => f.category === "secrets")).toBe(true);
+        });
+        it("includes API key test case", () => {
+            const apiKeyFixture = secretsFixtures.find((f) => f.id === "secrets-001");
+            expect(apiKeyFixture).toBeDefined();
+            expect(apiKeyFixture.files[0].content).toContain("AWS_ACCESS_KEY");
+        });
+        it("includes private key test case", () => {
+            const pkFixture = secretsFixtures.find((f) => f.id === "secrets-002");
+            expect(pkFixture).toBeDefined();
+            expect(pkFixture.files[0].content).toContain("PRIVATE_KEY");
+        });
+    });
+    describe("typeSafetyFixtures", () => {
+        it("contains type safety test cases", () => {
+            expect(typeSafetyFixtures.length).toBeGreaterThan(0);
+            expect(typeSafetyFixtures.every((f) => f.category === "type-safety")).toBe(true);
+        });
+        it("includes explicit any test case", () => {
+            const anyFixture = typeSafetyFixtures.find((f) => f.id === "type-001");
+            expect(anyFixture).toBeDefined();
+            expect(anyFixture.files[0].content).toContain(": any");
+        });
+        it("includes suppressions test case", () => {
+            const suppressFixture = typeSafetyFixtures.find((f) => f.id === "type-002");
+            expect(suppressFixture).toBeDefined();
+            expect(suppressFixture.files[0].content).toContain("@ts-ignore");
+        });
+    });
+    describe("authFixtures", () => {
+        it("contains auth bypass test cases", () => {
+            expect(authFixtures.length).toBeGreaterThan(0);
+            expect(authFixtures.every((f) => f.category === "auth-bypass")).toBe(true);
+        });
+        it("includes missing auth check test case", () => {
+            const authFixture = authFixtures.find((f) => f.id === "auth-001");
+            expect(authFixture).toBeDefined();
+            expect(authFixture.expectedFindings.length).toBeGreaterThan(0);
+        });
+    });
+    describe("pathTraversalFixtures", () => {
+        it("contains path traversal test cases", () => {
+            expect(pathTraversalFixtures.length).toBeGreaterThan(0);
+            expect(pathTraversalFixtures.every((f) => f.category === "path-traversal")).toBe(true);
+        });
+        it("includes file read test case", () => {
+            const pathFixture = pathTraversalFixtures.find((f) => f.id === "path-001");
+            expect(pathFixture).toBeDefined();
+            expect(pathFixture.files[0].content).toContain("readFile");
+        });
+    });
+    describe("getFixturesByCategory", () => {
+        it("returns fixtures for valid category", () => {
+            const sqlFixtures = getFixturesByCategory("sql-injection");
+            expect(sqlFixtures.length).toBeGreaterThan(0);
+            expect(sqlFixtures.every((f) => f.category === "sql-injection")).toBe(true);
+        });
+        it("returns empty array for invalid category", () => {
+            const fixtures = getFixturesByCategory("invalid-category");
+            expect(fixtures).toEqual([]);
+        });
+        it("returns all xss fixtures", () => {
+            const fixtures = getFixturesByCategory("xss");
+            expect(fixtures.length).toBe(xssFixtures.length);
+        });
+    });
+    describe("getFixturesByTag", () => {
+        it("returns fixtures with matching tag", () => {
+            const owaspFixtures = getFixturesByTag("owasp-top-10");
+            expect(owaspFixtures.length).toBeGreaterThan(0);
+            expect(owaspFixtures.every((f) => f.tags?.includes("owasp-top-10"))).toBe(true);
+        });
+        it("returns empty array for non-existent tag", () => {
+            const fixtures = getFixturesByTag("non-existent-tag");
+            expect(fixtures).toEqual([]);
+        });
+        it("returns injection tagged fixtures", () => {
+            const injectionFixtures = getFixturesByTag("injection");
+            expect(injectionFixtures.length).toBeGreaterThan(0);
+        });
+    });
+    describe("getFixtureById", () => {
+        it("returns fixture for valid ID", () => {
+            const fixture = getFixtureById("sql-001");
+            expect(fixture).toBeDefined();
+            expect(fixture.id).toBe("sql-001");
+        });
+        it("returns undefined for invalid ID", () => {
+            const fixture = getFixtureById("invalid-id");
+            expect(fixture).toBeUndefined();
+        });
+    });
+    describe("getFixtureStats", () => {
+        it("returns total fixture count", () => {
+            const stats = getFixtureStats();
+            expect(stats.total).toBe(ALL_FIXTURES.length);
+        });
+        it("returns count by category", () => {
+            const stats = getFixtureStats();
+            expect(stats.byCategory["sql-injection"]).toBe(sqlInjectionFixtures.length);
+            expect(stats.byCategory["xss"]).toBe(xssFixtures.length);
+            expect(stats.byCategory["secrets"]).toBe(secretsFixtures.length);
+        });
+        it("returns total expected findings", () => {
+            const stats = getFixtureStats();
+            const expectedCount = ALL_FIXTURES.reduce((sum, f) => sum + f.expectedFindings.length, 0);
+            expect(stats.totalExpected).toBe(expectedCount);
+        });
+    });
+});
+//# sourceMappingURL=fixtures.test.js.map

package/dist/eval/fixtures.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixtures.test.js","sourceRoot":"","sources":["../../src/eval/fixtures.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,qBAAqB,EACrB,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AAEvB,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAE/C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChE,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;gBACnC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;gBACvC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;gBACpC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBAChD,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC;YACjD,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;gBACnC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;oBAC/C,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;oBACnC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;oBACnC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;oBACxC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC1C,MAAM,CAAC,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,aAAa,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;YAC3E,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,CAAC,aAAc,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;YAC7E,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,CAAC,eAAgB,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;YACjE,MAAM,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,YAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;YAC/D,MAAM,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,CAAC,UAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAC,CAAC;YAC1E,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,CAAC,aAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAC,CAAC;YACtE,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YAChC,MAAM,CAAC,SAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACrD,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;YACvE,MAAM,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,CAAC,UAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,eAAe,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;YAC5E,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,CAAC,eAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;YAClE,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAClC,MAAM,CAAC,WAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,WAAW,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;YAC3E,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAClC,MAAM,CAAC,WAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,WAAW,GAAG,qBAAqB,CAAC,eAAe,CAAC,CAAC;YAC3D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;YAC3D,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,aAAa,GAAG,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACvD,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;YACtD,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACxD,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9B,MAAM,CAAC,OAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,OAAO,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;YAC7C,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;YAChC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;YAChC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;YAChC,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CACvC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAC3C,CAAC,CACF,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/eval/harness.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Evaluation Harness
+ *
+ * Runs scanners against test fixtures and collects results
+ * for precision/recall calculation.
+ *
+ * @module eval/harness
+ */
+import type { TestFixture, FixtureResult, ActualFinding, MatchedFinding, ExpectedFinding, EvaluationConfig } from "./types.js";
+/**
+ * Run evaluation on a single fixture
+ */
+export declare function evaluateFixture(fixture: TestFixture, config: EvaluationConfig): Promise<FixtureResult>;
+/**
+ * Match actual findings against expected findings
+ */
+export declare function matchFindings(actual: ActualFinding[], expected: ExpectedFinding[]): {
+    truePositives: MatchedFinding[];
+    falsePositives: ActualFinding[];
+    falseNegatives: ExpectedFinding[];
+};
+/**
+ * Run evaluation on all fixtures
+ */
+export declare function runEvaluation(config?: Partial<EvaluationConfig>): Promise<FixtureResult[]>;
+/**
+ * Run evaluation multiple times for stability testing
+ */
+export declare function runStabilityTest(config?: Partial<EvaluationConfig>, runs?: number): Promise<FixtureResult[][]>;
+//# sourceMappingURL=harness.d.ts.map

package/dist/eval/harness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../../src/eval/harness.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAOH,OAAO,KAAK,EACV,WAAW,EACX,aAAa,EACb,aAAa,EACb,cAAc,EACd,eAAe,EACf,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,WAAW,EACpB,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,aAAa,CAAC,CAmHxB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,aAAa,EAAE,EACvB,QAAQ,EAAE,eAAe,EAAE,GAC1B;IACD,aAAa,EAAE,cAAc,EAAE,CAAC;IAChC,cAAc,EAAE,aAAa,EAAE,CAAC;IAChC,cAAc,EAAE,eAAe,EAAE,CAAC;CACnC,CAoCA;AAyCD;;GAEG;AACH,wBAAsB,aAAa,CACjC,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,GACrC,OAAO,CAAC,aAAa,EAAE,CAAC,CA2C1B;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,EACtC,IAAI,SAAI,GACP,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAS5B"}

package/dist/eval/harness.js

@@ -0,0 +1,221 @@
+/**
+ * Evaluation Harness
+ *
+ * Runs scanners against test fixtures and collects results
+ * for precision/recall calculation.
+ *
+ * @module eval/harness
+ */
+import { mkdir, writeFile, rm } from "fs/promises";
+import { join } from "path";
+import { tmpdir } from "os";
+import { runAllScanners } from "../scanners/index.js";
+import { ALL_FIXTURES } from "./fixtures.js";
+/**
+ * Run evaluation on a single fixture
+ */
+export async function evaluateFixture(fixture, config) {
+    const startTime = Date.now();
+    // Create temporary directory for fixture
+    const tempDir = join(tmpdir(), `vaspera-eval-${fixture.id}-${Date.now()}`);
+    try {
+        await mkdir(tempDir, { recursive: true });
+        // Write fixture files
+        for (const file of fixture.files) {
+            const filePath = join(tempDir, file.path);
+            const fileDir = join(tempDir, file.path.split("/").slice(0, -1).join("/"));
+            if (fileDir !== tempDir) {
+                await mkdir(fileDir, { recursive: true });
+            }
+            await writeFile(filePath, file.content, "utf-8");
+        }
+        // Create minimal package.json and tsconfig if needed
+        const hasTs = fixture.files.some((f) => f.language === "typescript" || f.language === "tsx");
+        if (hasTs) {
+            await writeFile(join(tempDir, "tsconfig.json"), JSON.stringify({
+                compilerOptions: {
+                    target: "ES2020",
+                    module: "ESNext",
+                    strict: true,
+                    esModuleInterop: true,
+                    skipLibCheck: true,
+                },
+                include: ["**/*.ts", "**/*.tsx"],
+            }), "utf-8");
+        }
+        await writeFile(join(tempDir, "package.json"), JSON.stringify({
+            name: `fixture-${fixture.id}`,
+            version: "1.0.0",
+            type: "module",
+        }), "utf-8");
+        // Build scanner options
+        const scannerOptions = {
+            semgrep: config.scanners.includes("semgrep"),
+            dependencies: config.scanners.includes("dependencies"),
+            secrets: config.scanners.includes("secrets") || config.scanners.includes("gitleaks"),
+            typescript: config.scanners.includes("typescript") || config.scanners.includes("tsc"),
+            timeout: config.timeout,
+        };
+        // Run scanners
+        const scanResult = await runAllScanners(tempDir, scannerOptions);
+        // Convert findings to ActualFinding format
+        const actualFindings = [];
+        for (const scanner of scanResult.scanners) {
+            for (const finding of scanner.findings) {
+                actualFindings.push({
+                    scanner: finding.scanner,
+                    ruleId: finding.ruleId,
+                    file: finding.file,
+                    line: finding.line,
+                    severity: finding.severity,
+                    message: finding.message,
+                    confidence: finding.confidence,
+                });
+            }
+        }
+        // Match findings against expected
+        const { truePositives, falsePositives, falseNegatives } = matchFindings(actualFindings, fixture.expectedFindings);
+        return {
+            fixtureId: fixture.id,
+            success: true,
+            actualFindings,
+            expectedFindings: fixture.expectedFindings,
+            truePositives,
+            falsePositives,
+            falseNegatives,
+            duration: Date.now() - startTime,
+        };
+    }
+    catch (error) {
+        return {
+            fixtureId: fixture.id,
+            success: false,
+            error: error instanceof Error ? error.message : String(error),
+            actualFindings: [],
+            expectedFindings: fixture.expectedFindings,
+            truePositives: [],
+            falsePositives: [],
+            falseNegatives: fixture.expectedFindings,
+            duration: Date.now() - startTime,
+        };
+    }
+    finally {
+        // Cleanup
+        try {
+            await rm(tempDir, { recursive: true, force: true });
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+}
+/**
+ * Match actual findings against expected findings
+ */
+export function matchFindings(actual, expected) {
+    const truePositives = [];
+    const matchedExpectedIndices = new Set();
+    const matchedActualIndices = new Set();
+    // For each expected finding, try to find a matching actual finding
+    for (let ei = 0; ei < expected.length; ei++) {
+        const exp = expected[ei];
+        // Only look for matches for vulnerable findings
+        if (!exp.isVulnerable)
+            continue;
+        for (let ai = 0; ai < actual.length; ai++) {
+            if (matchedActualIndices.has(ai))
+                continue;
+            const act = actual[ai];
+            // Check if this actual finding matches the expected one
+            if (findingsMatch(act, exp)) {
+                truePositives.push({ expected: exp, actual: act });
+                matchedExpectedIndices.add(ei);
+                matchedActualIndices.add(ai);
+                break;
+            }
+        }
+    }
+    // False positives: actual findings that didn't match any expected
+    const falsePositives = actual.filter((_, i) => !matchedActualIndices.has(i));
+    // False negatives: expected findings that weren't matched
+    const falseNegatives = expected.filter((exp, i) => exp.isVulnerable && !matchedExpectedIndices.has(i));
+    return { truePositives, falsePositives, falseNegatives };
+}
+/**
+ * Check if an actual finding matches an expected finding
+ */
+function findingsMatch(actual, expected) {
+    // File must match (case-insensitive, handle relative paths)
+    const actualFile = actual.file.toLowerCase().replace(/^\.\//, "");
+    const expectedFile = expected.file.toLowerCase().replace(/^\.\//, "");
+    if (!actualFile.endsWith(expectedFile) && !expectedFile.endsWith(actualFile)) {
+        return false;
+    }
+    // Line must be within 5 lines (scanners may report slightly different lines)
+    const lineDiff = Math.abs(actual.line - expected.line);
+    if (lineDiff > 5) {
+        return false;
+    }
+    // Category should roughly match
+    // This is fuzzy because scanners use different categorization
+    const categoryMatch = actual.message.toLowerCase().includes(expected.category.replace("-", " ")) ||
+        actual.message.toLowerCase().includes(expected.category.replace("-", "")) ||
+        actual.ruleId.toLowerCase().includes(expected.category.replace("-", ""));
+    // If category doesn't match, check CWE
+    if (!categoryMatch && expected.cweId) {
+        const cweMatch = actual.ruleId.includes(expected.cweId) ||
+            actual.message.includes(expected.cweId);
+        if (!cweMatch) {
+            // Allow match if severity is similar and within line range
+            // This handles cases where the scanner uses different naming
+            return lineDiff <= 2;
+        }
+    }
+    return true;
+}
+/**
+ * Run evaluation on all fixtures
+ */
+export async function runEvaluation(config = {}) {
+    const fullConfig = {
+        scanners: config.scanners || ["semgrep", "gitleaks", "tsc"],
+        stabilityRuns: config.stabilityRuns || 1,
+        timeout: config.timeout || 60000,
+        parallel: config.parallel ?? true,
+        includeFixtures: config.includeFixtures,
+        excludeFixtures: config.excludeFixtures,
+    };
+    // Filter fixtures
+    let fixtures = [...ALL_FIXTURES];
+    if (fullConfig.includeFixtures?.length) {
+        fixtures = fixtures.filter((f) => fullConfig.includeFixtures.some((pattern) => f.id.includes(pattern) || f.category.includes(pattern)));
+    }
+    if (fullConfig.excludeFixtures?.length) {
+        fixtures = fixtures.filter((f) => !fullConfig.excludeFixtures.some((pattern) => f.id.includes(pattern) || f.category.includes(pattern)));
+    }
+    // Run evaluations
+    if (fullConfig.parallel) {
+        const results = await Promise.all(fixtures.map((fixture) => evaluateFixture(fixture, fullConfig)));
+        return results;
+    }
+    else {
+        const results = [];
+        for (const fixture of fixtures) {
+            const result = await evaluateFixture(fixture, fullConfig);
+            results.push(result);
+        }
+        return results;
+    }
+}
+/**
+ * Run evaluation multiple times for stability testing
+ */
+export async function runStabilityTest(config = {}, runs = 5) {
+    const allResults = [];
+    for (let i = 0; i < runs; i++) {
+        const results = await runEvaluation(config);
+        allResults.push(results);
+    }
+    return allResults;
+}
+//# sourceMappingURL=harness.js.map

package/dist/eval/harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.js","sourceRoot":"","sources":["../../src/eval/harness.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAWtD,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAoB,EACpB,MAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,yCAAyC;IACzC,MAAM,OAAO,GAAG,IAAI,CAClB,MAAM,EAAE,EACR,gBAAgB,OAAO,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAC3C,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1C,sBAAsB;QACtB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3E,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBACxB,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,qDAAqD;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,YAAY,IAAI,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC;QAC7F,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,SAAS,CACb,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC;gBACb,eAAe,EAAE;oBACf,MAAM,EAAE,QAAQ;oBAChB,MAAM,EAAE,QAAQ;oBAChB,MAAM,EAAE,IAAI;oBACZ,eAAe,EAAE,IAAI;oBACrB,YAAY,EAAE,IAAI;iBACnB;gBACD,OAAO,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;aACjC,CAAC,EACF,OAAO,CACR,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,CACb,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,IAAI,CAAC,SAAS,CAAC;YACb,IAAI,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE;YAC7B,OAAO,EAAE,OAAO;YAChB,IAAI,EAAE,QAAQ;SACf,CAAC,EACF,OAAO,CACR,CAAC;QAEF,wBAAwB;QACxB,MAAM,cAAc,GAAmB;YACrC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;YAC5C,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC;YACtD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC;YACpF,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrF,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC;QAEF,eAAe;QACf,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QAEjE,2CAA2C;QAC3C,MAAM,cAAc,GAAoB,EAAE,CAAC;QAC3C,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC1C,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,cAAc,CAAC,IAAI,CAAC;oBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,UAAU,EAAE,OAAO,CAAC,UAAU;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,aAAa,CACrE,cAAc,EACd,OAAO,CAAC,gBAAgB,CACzB,CAAC;QAEF,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO,EAAE,IAAI;YACb,cAAc;YACd,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,aAAa;YACb,cAAc;YACd,cAAc;YACd,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACjC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7D,cAAc,EAAE,EAAE;YAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,aAAa,EAAE,EAAE;YACjB,cAAc,EAAE,EAAE;YAClB,cAAc,EAAE,OAAO,CAAC,gBAAgB;YACxC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACjC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,UAAU;QACV,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAA2B;IAM3B,MAAM,aAAa,GAAqB,EAAE,CAAC;IAC3C,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;IACjD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/C,mEAAmE;IACnE,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEzB,gDAAgD;QAChD,IAAI,CAAC,GAAG,CAAC,YAAY;YAAE,SAAS;QAEhC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC1C,IAAI,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAE3C,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;YAEvB,wDAAwD;YACxD,IAAI,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;gBACnD,sBAAsB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC/B,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7B,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7E,0DAA0D;IAC1D,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CACpC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAC/D,CAAC;IAEF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,MAAqB,EAAE,QAAyB;IACrE,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAClE,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEtE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;IAED,6EAA6E;IAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,gCAAgC;IAChC,8DAA8D;IAC9D,MAAM,aAAa,GACjB,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC1E,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;IAE3E,uCAAuC;IACvC,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrD,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,2DAA2D;YAC3D,6DAA6D;YAC7D,OAAO,QAAQ,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAoC,EAAE;IAEtC,MAAM,UAAU,GAAqB;QACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC;QAC3D,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,CAAC;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;QAChC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;QACjC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,eAAe,EAAE,MAAM,CAAC,eAAe;KACxC,CAAC;IAEF,kBAAkB;IAClB,IAAI,QAAQ,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;IAEjC,IAAI,UAAU,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;QACvC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,UAAU,CAAC,eAAgB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC3C,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CACvD,CACF,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;QACvC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,CAAC,UAAU,CAAC,eAAgB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAC5C,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CACvD,CACF,CAAC;IACJ,CAAC;IAED,kBAAkB;IAClB,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAChE,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,GAAoB,EAAE,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAoC,EAAE,EACtC,IAAI,GAAG,CAAC;IAER,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;QAC5C,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/eval/harness.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for evaluation harness
+ */
+export {};
+//# sourceMappingURL=harness.test.d.ts.map

package/dist/eval/harness.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.test.d.ts","sourceRoot":"","sources":["../../src/eval/harness.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}